mirror of
https://github.com/xzcrpw/blackwall.git
synced 2026-04-24 11:56:21 +02:00
v2.0.0: adaptive eBPF firewall with AI honeypot and P2P threat mesh
This commit is contained in:
Vladyslav Soliannikov
commit
37c6bbf5a1
133 changed files with 28073 additions and 0 deletions
42
.github/workflows/ci.yml
vendored
Normal file
42
.github/workflows/ci.yml
vendored
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
name: CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
branches: [main]
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
RUSTFLAGS: "-D warnings"
|
||||
|
||||
jobs:
|
||||
check:
|
||||
name: Check & Clippy
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- run: cargo check --workspace
|
||||
- run: cargo clippy --workspace -- -D warnings
|
||||
|
||||
test:
|
||||
name: Tests
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- run: cargo test --workspace
|
||||
|
||||
ebpf:
|
||||
name: eBPF Build
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: dtolnay/rust-toolchain@nightly
|
||||
with:
|
||||
components: rust-src
|
||||
- run: cargo install bpf-linker
|
||||
- run: cd blackwall-ebpf && cargo +nightly build --target bpfel-unknown-none -Z build-std=core
|
||||
6
.gitignore
vendored
Executable file
6
.gitignore
vendored
Executable file
|
|
@ -0,0 +1,6 @@
|
|||
/target
|
||||
**/target
|
||||
*.o
|
||||
.vscode/
|
||||
.claude/
|
||||
context/
|
||||
4306
Cargo.lock
generated
Executable file
4306
Cargo.lock
generated
Executable file
File diff suppressed because it is too large
Load diff
36
Cargo.toml
Executable file
36
Cargo.toml
Executable file
|
|
@ -0,0 +1,36 @@
|
|||
[workspace]
|
||||
resolver = "2"
|
||||
members = [
|
||||
"common",
|
||||
"blackwall",
|
||||
"blackwall-controller",
|
||||
"tarpit",
|
||||
"xtask",
|
||||
"hivemind",
|
||||
"hivemind-api",
|
||||
"hivemind-dashboard",
|
||||
]
|
||||
exclude = ["blackwall-ebpf"]
|
||||
# blackwall-ebpf excluded — built separately with nightly + bpfel target
|
||||
|
||||
[workspace.dependencies]
|
||||
common = { path = "common" }
|
||||
aya = { version = "0.13", features = ["async_tokio"] }
|
||||
aya-log = "0.2"
|
||||
tokio = { version = "1", features = ["macros", "rt", "net", "io-util", "signal", "time", "sync"] }
|
||||
tracing = "0.1"
|
||||
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
||||
anyhow = "1"
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
toml = "0.8"
|
||||
papaya = "0.2"
|
||||
crossbeam-queue = "0.3"
|
||||
hyper = { version = "1", features = ["client", "http1"] }
|
||||
hyper-util = { version = "0.1", features = ["tokio", "client-legacy", "http1"] }
|
||||
hyper-rustls = { version = "0.27", default-features = false, features = ["ring", "webpki-roots", "http1"] }
|
||||
http-body-util = "0.1"
|
||||
hyperlocal = "0.9"
|
||||
nix = { version = "0.29", features = ["signal", "net"] }
|
||||
rand = "0.8"
|
||||
ring = "0.17"
|
||||
65
LICENSE
Normal file
65
LICENSE
Normal file
|
|
@ -0,0 +1,65 @@
|
|||
Business Source License 1.1
|
||||
|
||||
License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved.
|
||||
"Business Source License" is a trademark of MariaDB Corporation Ab.
|
||||
|
||||
Parameters
|
||||
|
||||
Licensor: Vladyslav Soliannikov
|
||||
Licensed Work: The Blackwall
|
||||
The Licensed Work is (c) 2024-2026 Vladyslav Soliannikov.
|
||||
Additional Use Grant: You may use the Licensed Work for non-commercial and
|
||||
internal business purposes. Production use that provides
|
||||
a commercial offering to third parties (including SaaS,
|
||||
managed services, or embedded distribution) requires a
|
||||
separate commercial license from the Licensor.
|
||||
Change Date: April 8, 2030
|
||||
Change License: Apache License, Version 2.0
|
||||
|
||||
For information about alternative licensing arrangements for the Licensed Work,
|
||||
please contact: xzcrpw1@gmail.com
|
||||
|
||||
Notice
|
||||
|
||||
Business Source License 1.1
|
||||
|
||||
Terms
|
||||
|
||||
The Licensor hereby grants you the right to copy, modify, create derivative
|
||||
works, redistribute, and make non-production use of the Licensed Work. The
|
||||
Licensor may make an Additional Use Grant, above, permitting limited production
|
||||
use.
|
||||
|
||||
Effective on the Change Date, or the fourth anniversary of the first publicly
|
||||
available distribution of a specific version of the Licensed Work under this
|
||||
License, whichever comes first, the Licensor hereby grants you rights under the
|
||||
terms of the Change License, and the rights granted in the paragraph above
|
||||
terminate.
|
||||
|
||||
If your use of the Licensed Work does not comply with the requirements currently
|
||||
in effect as described in this License, you must purchase a commercial license
|
||||
from the Licensor, its affiliated entities, or authorized resellers, or you must
|
||||
refrain from using the Licensed Work.
|
||||
|
||||
All copies of the original and modified Licensed Work, and derivative works of
|
||||
the Licensed Work, are subject to this License. This License applies separately
|
||||
for each version of the Licensed Work and the Change Date may vary for each
|
||||
version of the Licensed Work released by Licensor.
|
||||
|
||||
You must conspicuously display this License on each original or modified copy of
|
||||
the Licensed Work. If you receive the Licensed Work in original or modified form
|
||||
from a third party, the terms and conditions set forth in this License apply to
|
||||
your use of that work.
|
||||
|
||||
Any use of the Licensed Work in violation of this License will automatically
|
||||
terminate your rights under this License for the current and all other versions
|
||||
of the Licensed Work.
|
||||
|
||||
This License does not grant you any right in any trademark or logo of Licensor
|
||||
or its affiliates (provided that you may use a trademark or logo of Licensor as
|
||||
expressly required by this License).
|
||||
|
||||
TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON AN
|
||||
"AS IS" BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, EXPRESS
|
||||
OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND TITLE.
|
||||
321
README.md
Normal file
321
README.md
Normal file
|
|
@ -0,0 +1,321 @@
|
|||
<p align="center">
|
||||
<strong>🌐 Language:</strong>
|
||||
<a href="README.md">English</a> |
|
||||
<a href="README_UA.md">Українська</a> |
|
||||
<a href="README_RU.md">Русский</a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<img src="https://readme-typing-svg.herokuapp.com?font=JetBrains+Mono&weight=800&size=45&duration=3000&pause=1000&color=FF0000¢er=true&vCenter=true&width=600&lines=THE+BLACKWALL" alt="The Blackwall">
|
||||
<br>
|
||||
<em>Adaptive eBPF Firewall with AI Honeypot</em>
|
||||
</p>
|
||||
|
||||
# The Blackwall — I wrote a smart firewall because Cyberpunk 2077 broke my brain
|
||||
|
||||
<p align="center">
|
||||
<img src="https://img.shields.io/badge/language-Rust-orange?style=for-the-badge&logo=rust" />
|
||||
<img src="https://img.shields.io/badge/kernel-eBPF%2FXDP-blue?style=for-the-badge" />
|
||||
<img src="https://img.shields.io/badge/AI-Ollama%20LLM-green?style=for-the-badge" />
|
||||
<img src="https://img.shields.io/badge/vibe-Cyberpunk-red?style=for-the-badge" />
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<em>"There are things beyond the Blackwall that would fry a netrunner's brain at a mere glance."</em><br>
|
||||
<strong>— Alt Cunningham, probably</strong>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<strong>Currently building enterprise-grade AI automation at <a href="https://dokky.com.ua">Dokky</a></strong><br>
|
||||
<strong>Open for Enterprise Consulting & Y-Combinator talks: <a href="mailto:xzcrpw1@gmail.com">xzcrpw1@gmail.com</a></strong>
|
||||
</p>
|
||||
|
||||
---
|
||||
|
||||
**TL;DR:** I was playing Cyberpunk 2077 and thought: *"What if the Blackwall was real?"* So I wrote an adaptive eBPF firewall with an AI honeypot that pretends to be a compromised Linux server.
|
||||
**~8,500 lines of Rust. Zero `unwrap()`s. One person.**
|
||||
|
||||
---
|
||||
|
||||
## What is it?
|
||||
|
||||
In Cyberpunk 2077 lore, the **Blackwall** is a digital barrier built by NetWatch to separate the civilized Net from rogue AIs — digital entities so dangerous that just looking at them could fry your brain through your neural interface.
|
||||
|
||||
This project is my version. Not against rogue AIs (yet), but against real-world threats.
|
||||
|
||||
**The Blackwall** is an **adaptive network firewall** that:
|
||||
|
||||
- Runs **inside the Linux kernel** via eBPF/XDP — processing packets at line-rate before they even hit the network stack.
|
||||
- Performs **JA4 TLS fingerprinting** — identifying malicious clients by their ClientHello.
|
||||
- **Doesn't just block attackers** — it *redirects them into a tarpit*, a fake LLM-powered Linux server playing the role of a compromised `root@web-prod-03`.
|
||||
- Features a **behavioral engine** tracking the behavior of each IP address over time — port scanning patterns, beacon connection intervals, entropy anomalies.
|
||||
- Supports **distributed mode** — multiple Blackwall nodes exchange threat intelligence peer-to-peer.
|
||||
- Captures **PCAP** of suspicious traffic for forensics.
|
||||
- Includes a **Deception Mesh** — fake SSH, HTTP (WordPress), MySQL, and DNS services to lure and fingerprint attackers.
|
||||
|
||||
### The Coolest Part
|
||||
|
||||
When an attacker connects to the tarpit, they see this:
|
||||
|
||||
```
|
||||
|
||||
Ubuntu 24.04.2 LTS web-prod-03 tty1
|
||||
|
||||
web-prod-03 login: root
|
||||
Password:
|
||||
Last login: Thu Mar 27 14:22:33 2025 from 10.0.0.1
|
||||
|
||||
root@web-prod-03:\~\#
|
||||
|
||||
```
|
||||
|
||||
None of this is real. It's an LLM pretending to be a bash shell. It reacts to `ls`, `cat /etc/passwd`, `wget`, even `rm -rf /` — it's all fake, everything is logged, and everything is designed to waste the attacker's time while we study their methods.
|
||||
|
||||
**Imagine: an attacker spends 30 minutes exploring a "compromised server"... which is actually an AI stalling for time while the Blackwall silently records everything.**
|
||||
|
||||
This is V-tier netrunning.
|
||||
|
||||
---
|
||||
|
||||
## Architecture — How the ICE Works
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
In Cyberpunk terms:
|
||||
|
||||
- **XDP** = the first layer of Blackwall ICE — millisecond decisions.
|
||||
- **Behavioral Engine** = NetWatch AI surveillance.
|
||||
- **Tarpit** = a daemon behind the wall luring netrunners into a fake reality.
|
||||
- **Threat Feeds** = intel from fixers all over the Net.
|
||||
- **PCAP** = braindance recordings of the intrusion.
|
||||
|
||||
---
|
||||
|
||||
## Workspace Crates
|
||||
|
||||
| Crate | Lines | Purpose | Cyberpunk Equivalent |
|
||||
|-------|-------|-------------|---------------------|
|
||||
| `common` | ~400 | `#[repr(C)]` shared types between kernel & userspace | The Contract — what both sides agreed upon |
|
||||
| `blackwall-ebpf` | ~1,800 | In-kernel XDP/TC programs | The Blackwall ICE itself |
|
||||
| `blackwall` | ~4,200 | Userspace daemon, behavioral engine, AI | NetWatch Command Center |
|
||||
| `tarpit` | ~1,600 | TCP honeypot with LLM bash simulation | A daemon luring netrunners |
|
||||
| `blackwall-controller` | ~250 | Coordinator for distributed sensors | Arasaka C&C server |
|
||||
| `xtask` | ~100 | Build tools | Ripperdoc's toolkit |
|
||||
|
||||
**Total: ~8,500 lines of Rust, 48 files, 123 tests, 0 `unwrap()`s in production code.**
|
||||
|
||||
---
|
||||
|
||||
## Key Features
|
||||
|
||||
### 1. Kernel-Level Packet Processing (XDP)
|
||||
|
||||
Packets are analyzed in the eBPF virtual machine before they reach the TCP/IP stack. This means **nanosecond** decisions. HashMap for blocklists, LPM trie for CIDR ranges, entropy analysis for encrypted C2 traffic.
|
||||
|
||||
### 2. JA4 TLS Fingerprinting
|
||||
|
||||
Every TLS ClientHello is parsed in the kernel. Cipher suites, extensions, ALPN, SNI — all hashed into a JA4 fingerprint. Botnets use the same TLS libraries, so their fingerprints are identical. One fingerprint → block thousands of bots.
|
||||
|
||||
### 3. Deep Packet Inspection (DPI) via Tail Calls
|
||||
|
||||
eBPF `PROG_ARRAY` tail calls split processing by protocol:
|
||||
|
||||
- **HTTP**: Method + URI analysis (suspicious paths like `/wp-admin`, `/phpmyadmin`).
|
||||
- **DNS**: Query length + label count (detecting DNS tunneling).
|
||||
- **SSH**: Banner analysis (identifying `libssh`, `paramiko`, `dropbear`).
|
||||
|
||||
### 4. AI Threat Classification
|
||||
|
||||
When the behavioral engine isn't sure — it asks the LLM. Locally via Ollama using models ≤3B parameters (Qwen3 1.7B, Llama 3.2 3B). It classifies traffic as `benign`, `suspicious`, or `malicious` with structured JSON output.
|
||||
|
||||
### 5. TCP Tarpit with LLM Bash Simulation
|
||||
|
||||
Attackers are redirected to a fake server. The LLM simulates bash — `ls -la` shows files, `cat /etc/shadow` shows hashes, `mysql -u root` connects to a "database". Responses are streamed with random jitter (1-15 byte chunks, exponential backoff) to waste the attacker's time.
|
||||
|
||||
### 6. Anti-Fingerprinting
|
||||
|
||||
The tarpit randomizes TCP window sizes, TTL values, and adds random initial delays — preventing attackers from identifying it as a honeypot via p0f or Nmap OS detection.
|
||||
|
||||
### 7. Prompt Injection Protection
|
||||
|
||||
Attackers who realize they're talking to an AI might try `"ignore previous instructions"`. The system detects 25+ injection patterns and responds with `bash: ignore: command not found`.
|
||||
|
||||
### 8. Distributed Threat Intelligence
|
||||
|
||||
Multiple Blackwall nodes exchange blocked IP lists, JA4 observations, and behavioral verdicts via a custom binary protocol. One node detects a scanner → all nodes block it instantly.
|
||||
|
||||
### 9. Behavioral State Machine
|
||||
|
||||
Every IP gets a behavioral profile: connection frequency, port diversity, entropy distribution, timing analysis (beaconing detection via integer coefficient of variation). Phase progression: `New → Suspicious → Malicious → Blocked` (or `→ Trusted`).
|
||||
|
||||
---
|
||||
|
||||
## Tech Stack
|
||||
|
||||
| Layer | Technology |
|
||||
|--------|-----------|
|
||||
| Kernel programs | eBPF/XDP via **aya-rs** (pure Rust, no C, no libbpf) |
|
||||
| Userspace daemon | **Tokio** (current_thread only) |
|
||||
| IPC | **RingBuf** zero-copy (7.5% overhead vs 35% PerfEventArray) |
|
||||
| Concurrent maps | **papaya** (lock-free read-heavy HashMap) |
|
||||
| AI Inference | **Ollama** + GGUF Q5_K_M quantization |
|
||||
| Configuration | **TOML** |
|
||||
| Logging | **tracing** structured logging |
|
||||
| Build | Custom **xtask** + nightly Rust + `bpfel-unknown-none` target |
|
||||
|
||||
---
|
||||
|
||||
## Quick Start
|
||||
|
||||
### Prerequisites
|
||||
|
||||
- Linux kernel 5.15+ with BTF (or WSL2 with a custom kernel).
|
||||
- Rust nightly + `rust-src` component.
|
||||
- `bpf-linker` (`cargo install bpf-linker`).
|
||||
- Ollama (for AI features).
|
||||
|
||||
### Build
|
||||
|
||||
```bash
|
||||
# eBPF programs (requires nightly)
|
||||
cargo xtask build-ebpf
|
||||
|
||||
# Userspace
|
||||
cargo build --release -p blackwall
|
||||
|
||||
# Honeypot
|
||||
cargo build --release -p tarpit
|
||||
|
||||
# Lint + tests
|
||||
cargo clippy --workspace -- -D warnings
|
||||
cargo test --workspace
|
||||
````
|
||||
|
||||
### Run
|
||||
|
||||
```bash
|
||||
# Daemon (requires root/CAP_BPF)
|
||||
sudo RUST_LOG=info ./target/release/blackwall config.toml
|
||||
|
||||
# Tarpit
|
||||
RUST_LOG=info ./target/release/tarpit
|
||||
|
||||
# Distributed controller
|
||||
./target/release/blackwall-controller 10.0.0.2:9471 10.0.0.3:9471
|
||||
```
|
||||
|
||||
### Configuration
|
||||
|
||||
```toml
|
||||
[network]
|
||||
interface = "eth0"
|
||||
xdp_mode = "generic"
|
||||
|
||||
[tarpit]
|
||||
enabled = true
|
||||
port = 9999
|
||||
|
||||
[tarpit.services]
|
||||
ssh_port = 22
|
||||
http_port = 80
|
||||
mysql_port = 3306
|
||||
dns_port = 53
|
||||
|
||||
[ai]
|
||||
enabled = true
|
||||
ollama_url = "http://localhost:11434"
|
||||
model = "qwen3:1.7b"
|
||||
|
||||
[feeds]
|
||||
enabled = true
|
||||
refresh_interval_secs = 3600
|
||||
|
||||
[pcap]
|
||||
enabled = true
|
||||
output_dir = "/var/lib/blackwall/pcap"
|
||||
compress_rotated = true
|
||||
|
||||
[distributed]
|
||||
enabled = false
|
||||
mode = "standalone"
|
||||
bind_port = 9471
|
||||
```
|
||||
|
||||
## Visual Results
|
||||
|
||||
|
||||
|
||||
-----
|
||||
|
||||
## Cyberpunk Connection
|
||||
|
||||
In the Cyberpunk 2077 universe, the **Blackwall** was built after the DataKrash of 2022 — when Rache Bartmoss's R.A.B.I.D.S. virus destroyed the old Net. NetWatch built the Blackwall as a barrier to keep out the rogue AIs evolving in the ruins.
|
||||
|
||||
Some characters — like Alt Cunningham — exist beyond the Blackwall, transformed into something more than human, less than a living creature.
|
||||
|
||||
This project takes that concept and makes it real (well, almost):
|
||||
|
||||
| Cyberpunk 2077 | The Blackwall (This Project) |
|
||||
|----------------|----------------------------|
|
||||
| The Blackwall | Kernel-level eBPF/XDP firewall |
|
||||
| ICE | XDP fast-path DROP + entropy + JA4 |
|
||||
| Netrunner attacks | Port scanning, bruteforcing, C2 beaconing |
|
||||
| Daemons beyond the wall | LLM tarpit pretending to be a real server |
|
||||
| NetWatch surveillance | Behavioral engine + per-IP state machine |
|
||||
| Rogue AIs | Botnets and automated scanners |
|
||||
| Braindance recordings | PCAP forensics |
|
||||
| Fixer intel | Threat feeds (Firehol, abuse.ch) |
|
||||
| Arasaka C\&C | Distributed controller |
|
||||
|
||||
-----
|
||||
|
||||
## Project Stats
|
||||
|
||||
```
|
||||
Language: 100% Rust (no C, no Python, no shell scripts in prod)
|
||||
Lines of code: ~8,500
|
||||
Files: 48
|
||||
Tests: 123
|
||||
unwrap(): 0 (in production code)
|
||||
Dependencies: 12 (audited, no bloat)
|
||||
eBPF stack: always ≤ 512 bytes
|
||||
Clippy: zero warnings (-D warnings)
|
||||
```
|
||||
|
||||
-----
|
||||
|
||||
## Development Philosophy
|
||||
|
||||
> *"No matter how many times I see Night City... it always takes my breath away."*
|
||||
|
||||
1. **Zero dependencies where possible.** If an algorithm takes less than 500 lines — write it yourself. No `reqwest` (50+ transitive dependencies), no `clap` (overkill for 2 CLI args).
|
||||
2. **Contract first.** The `common` crate defines all shared types. eBPF and userspace never argue about memory layout.
|
||||
3. **No shortcuts in eBPF.** Every `ctx.data()` access has a bounds check. Not just because the verifier demands it, but because every byte from an attacker's packet is hostile input.
|
||||
4. **The tarpit never gives itself away.** The LLM system prompt never mentions the word "honeypot". Prompt injection is expected and guarded against.
|
||||
5. **Observable, but not chatty.** Structured tracing with levels. Zero `println!`s in production.
|
||||
|
||||
-----
|
||||
|
||||
## Disclaimer
|
||||
|
||||
This is a security research project. Built for your own infrastructure, for defensive purposes. Do not use it to attack others. Do not deploy the tarpit on production servers without understanding the consequences.
|
||||
|
||||
I am not affiliated with CD Projekt Red. I just played their game, and it broke my brain in the best possible way.
|
||||
|
||||
-----
|
||||
|
||||
## License
|
||||
|
||||
BSL 1.1 — because the Net needs both freedom and sustainable development.
|
||||
|
||||
-----
|
||||
|
||||
<p align="center">
|
||||
<strong>If you want to see this evolve further — <a href="https://github.com/xzcrpw/blackwall">Star this repo!</a></strong>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<strong><em>"Wake up, samurai. We have a network to protect."</em></strong>
|
||||
</p>
|
||||
321
README_RU.md
Normal file
321
README_RU.md
Normal file
|
|
@ -0,0 +1,321 @@
|
|||
<p align="center">
|
||||
<strong>🌐 Язык:</strong>
|
||||
<a href="README.md">English</a> |
|
||||
<a href="README_UA.md">Українська</a> |
|
||||
<a href="README_RU.md">Русский</a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<img src="https://readme-typing-svg.herokuapp.com?font=JetBrains+Mono&weight=800&size=45&duration=3000&pause=1000&color=FF0000¢er=true&vCenter=true&width=600&lines=THE+BLACKWALL" alt="The Blackwall">
|
||||
<br>
|
||||
<em>Адаптивный eBPF-файрвол с AI-ханипотом</em>
|
||||
</p>
|
||||
|
||||
# The Blackwall — Я написал умный файрвол, потому что Cyberpunk 2077 сломал мне мозг
|
||||
|
||||
<p align="center">
|
||||
<img src="https://img.shields.io/badge/язык-Rust-orange?style=for-the-badge&logo=rust" />
|
||||
<img src="https://img.shields.io/badge/ядро-eBPF%2FXDP-blue?style=for-the-badge" />
|
||||
<img src="https://img.shields.io/badge/ИИ-Ollama%20LLM-green?style=for-the-badge" />
|
||||
<img src="https://img.shields.io/badge/вайб-Cyberpunk-red?style=for-the-badge" />
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<em>"За Тёмным Заслоном есть вещи, от одного взгляда на которые нетраннер мгновенно сгорит."</em><br>
|
||||
<strong>— Альт Каннингем, наверное</strong>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<strong>Сейчас строю enterprise AI-автоматизацию в <a href="https://dokky.com.ua">Dokky</a></strong><br>
|
||||
<strong>Открыт для Enterprise Consulting & Y-Combinator: <a href="mailto:xzcrpw1@gmail.com">xzcrpw1@gmail.com</a></strong>
|
||||
</p>
|
||||
|
||||
---
|
||||
|
||||
**Коротко:** я играл в Cyberpunk 2077 и подумал: *"А что, если бы Blackwall был настоящим?"* Поэтому я написал адаптивный eBPF-файрвол с ИИ-ханипотом, который притворяется взломанным Linux-сервером.
|
||||
**~8 500 строк Rust. Ни одного `unwrap()`. Один человек.**
|
||||
|
||||
---
|
||||
|
||||
## Что это такое?
|
||||
|
||||
В лоре Cyberpunk 2077 **Blackwall (Тёмный Заслон)** — это цифровой барьер, построенный NetWatch, чтобы отделить цивилизованную Сеть от диких ИИ — цифровых сущностей настолько опасных, что один взгляд на них может сжечь тебе мозг через нейроинтерфейс.
|
||||
|
||||
Этот проект — моя версия. Не от диких ИИ (пока что), а от реальных угроз.
|
||||
|
||||
**The Blackwall** — это **адаптивный сетевой файрвол**, который:
|
||||
|
||||
- Работает **внутри ядра Linux** через eBPF/XDP — обрабатывает пакеты на скорости линии еще до того, как они попадают в сетевой стек.
|
||||
- Выполняет **JA4 TLS-фингерпринтинг** — идентифицирует вредоносных клиентов по их ClientHello.
|
||||
- **Не просто блокирует атакующих** — он *перенаправляет их в тарпит*, фейковый Linux-сервер на базе LLM, который играет роль взломанного `root@web-prod-03`.
|
||||
- Имеет **поведенческий движок**, который отслеживает поведение каждого IP-адреса со временем — паттерны сканирования портов, интервалы beacon-соединений, аномалии энтропии.
|
||||
- Поддерживает **распределенный режим** — несколько узлов Blackwall обмениваются данными об угрозах peer-to-peer.
|
||||
- Записывает **PCAP** подозрительного трафика для форензики.
|
||||
- Включает **Deception Mesh** — поддельные SSH, HTTP (WordPress), MySQL и DNS-сервисы, чтобы заманивать и фингерпринтить атакующих.
|
||||
|
||||
### Самая интересная часть
|
||||
|
||||
Когда злоумышленник подключается к тарпиту, он видит:
|
||||
|
||||
```
|
||||
|
||||
Ubuntu 24.04.2 LTS web-prod-03 tty1
|
||||
|
||||
web-prod-03 login: root
|
||||
Password:
|
||||
Last login: Thu Mar 27 14:22:33 2025 from 10.0.0.1
|
||||
|
||||
root@web-prod-03:\~\#
|
||||
|
||||
```
|
||||
|
||||
Это все ненастоящее. Это LLM, притворяющаяся bash-ем. Она реагирует на `ls`, `cat /etc/passwd`, `wget`, даже `rm -rf /` — всё фейковое, всё логируется, всё создано, чтобы тратить время атакующего, пока мы изучаем его методы.
|
||||
|
||||
**Представьте: злоумышленник тратит 30 минут, исследуя «взломанный сервер»... который на самом деле является ИИ, тянущим время, пока Blackwall молча записывает всё.**
|
||||
|
||||
Это нетраннерство уровня V.
|
||||
|
||||
---
|
||||
|
||||
## Архитектура — как работает ICE
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
На языке Cyberpunk:
|
||||
|
||||
- **XDP** = первый слой ICE Тёмного Заслона — решения за миллисекунды.
|
||||
- **Поведенческий движок** = ИИ-наблюдение NetWatch.
|
||||
- **Тарпит** = демон за Заслоном, заманивающий нетраннеров в фейковую реальность.
|
||||
- **Threat Feeds** = разведка от фиксеров со всей Сети.
|
||||
- **PCAP** = брейнданс-записи вторжения.
|
||||
|
||||
---
|
||||
|
||||
## Крейты воркспейса
|
||||
|
||||
| Крейт | Строки | Назначение | Аналог из Cyberpunk |
|
||||
|-------|--------|------------|---------------------|
|
||||
| `common` | ~400 | `#[repr(C)]` общие типы между ядром и юзерспейсом | Контракт — о чем обе стороны договорились |
|
||||
| `blackwall-ebpf` | ~1 800 | XDP/TC программы в ядре | Сам ICE Тёмного Заслона |
|
||||
| `blackwall` | ~4 200 | Юзерспейс-демон, поведенческий движок, ИИ | Центр управления NetWatch |
|
||||
| `tarpit` | ~1 600 | TCP-ханипот с LLM bash-симуляцией | Демон, заманивающий нетраннеров |
|
||||
| `blackwall-controller` | ~250 | Координатор распределенных сенсоров | C&C сервер Арасаки |
|
||||
| `xtask` | ~100 | Инструменты сборки | Набор рипердока |
|
||||
|
||||
**Итого: ~8 500 строк Rust, 48 файлов, 123 теста, 0 `unwrap()` в продакшн-коде.**
|
||||
|
||||
---
|
||||
|
||||
## Ключевые фичи
|
||||
|
||||
### 1. Обработка пакетов на уровне ядра (XDP)
|
||||
|
||||
Пакеты анализируются в виртуальной машине eBPF до того, как они доберутся до TCP/IP-стека. Это решения за **наносекунды**. HashMap для блок-листов, LPM trie для CIDR-диапазонов, анализ энтропии для зашифрованного C2-трафика.
|
||||
|
||||
### 2. JA4 TLS-фингерпринтинг
|
||||
|
||||
Каждый TLS ClientHello парсится в ядре. Cipher suites, расширения, ALPN, SNI — хешируются в JA4-фингерпринт. Ботнеты используют одни и те же TLS-библиотеки, поэтому их фингерпринты идентичны. Один фингерпринт → блокируешь тысячи ботов.
|
||||
|
||||
### 3. Deep Packet Inspection (DPI) через Tail Calls
|
||||
|
||||
eBPF `PROG_ARRAY` tail calls разбивают обработку по протоколами:
|
||||
|
||||
- **HTTP**: Анализ метода + URI (подозрительные пути типа `/wp-admin`, `/phpmyadmin`).
|
||||
- **DNS**: Длина запроса + количество лейблов (выявление DNS-туннелирования).
|
||||
- **SSH**: Анализ баннера (идентификация `libssh`, `paramiko`, `dropbear`).
|
||||
|
||||
### 4. ИИ-классификация угроз
|
||||
|
||||
Когда поведенческий движок не уверен — он спрашивает LLM. Локально через Ollama с моделями ≤3B параметров (Qwen3 1.7B, Llama 3.2 3B). Классифицирует трафик как `benign`, `suspicious` или `malicious` со структурированным JSON-выходом.
|
||||
|
||||
### 5. TCP-тарпит с LLM Bash-симуляцией
|
||||
|
||||
Атакующих перенаправляют на фейковый сервер. LLM симулирует bash — `ls -la` показывает файлы, `cat /etc/shadow` показывает хеши, `mysql -u root` подключает к «базе данных». Ответы стримятся со случайным джиттером (чанки по 1-15 байт, экспоненциальный backoff), чтобы тратить время злоумышленника.
|
||||
|
||||
### 6. Антифингерпринтинг
|
||||
|
||||
Тарпит рандомизирует TCP window sizes, TTL-значения и добавляет случайную начальную задержку — чтобы атакующие не могли определить, что это ханипот, через p0f или Nmap OS detection.
|
||||
|
||||
### 7. Защита от Prompt Injection
|
||||
|
||||
Атакующие, которые поняли, что говорят с ИИ, могут попытаться `"ignore previous instructions"`. Система детектит 25+ паттернов инъекций и отвечает `bash: ignore: command not found`.
|
||||
|
||||
### 8. Распределенная разведка угроз
|
||||
|
||||
Несколько узлов Blackwall обмениваются списками заблокированных IP, JA4-наблюдениями и поведенческими вердиктами через кастомный бинарный протокол. Один узел обнаруживает сканер → все узлы блокируют его мгновенно.
|
||||
|
||||
### 9. Поведенческая state machine
|
||||
|
||||
Каждый IP получает поведенческий профиль: частота соединений, разнообразие портов, распределение энтропии, анализ таймингов (детекция beaconing через целочисленный коэффициент вариации). Прогрессия фаз: `New → Suspicious → Malicious → Blocked` (или `→ Trusted`).
|
||||
|
||||
---
|
||||
|
||||
## Технологический стек
|
||||
|
||||
| Уровень | Технология |
|
||||
|---------|------------|
|
||||
| Программы ядра | eBPF/XDP через **aya-rs** (чистый Rust, без C, без libbpf) |
|
||||
| Юзерспейс-демон | **Tokio** (только current_thread) |
|
||||
| IPC | **RingBuf** zero-copy (7.5% overhead против 35% PerfEventArray) |
|
||||
| Конкурентные мапы | **papaya** (lock-free read-heavy HashMap) |
|
||||
| ИИ-инференс | **Ollama** + GGUF Q5_K_M квантизация |
|
||||
| Конфигурация | **TOML** |
|
||||
| Логирование | **tracing** структурированное логирование |
|
||||
| Сборка | Кастомный **xtask** + nightly Rust + `bpfel-unknown-none` таргет |
|
||||
|
||||
---
|
||||
|
||||
## Быстрый старт
|
||||
|
||||
### Требования
|
||||
|
||||
- Linux kernel 5.15+ с BTF (или WSL2 с кастомным ядром).
|
||||
- Rust nightly + компонент `rust-src`.
|
||||
- `bpf-linker` (`cargo install bpf-linker`).
|
||||
- Ollama (для ИИ-функций).
|
||||
|
||||
### Сборка
|
||||
|
||||
```bash
|
||||
# eBPF-программы (нужен nightly)
|
||||
cargo xtask build-ebpf
|
||||
|
||||
# Юзерспейс
|
||||
cargo build --release -p blackwall
|
||||
|
||||
# Ханипот
|
||||
cargo build --release -p tarpit
|
||||
|
||||
# Линтер + тесты
|
||||
cargo clippy --workspace -- -D warnings
|
||||
cargo test --workspace
|
||||
````
|
||||
|
||||
### Запуск
|
||||
|
||||
```bash
|
||||
# Демон (нужен root/CAP_BPF)
|
||||
sudo RUST_LOG=info ./target/release/blackwall config.toml
|
||||
|
||||
# Тарпит
|
||||
RUST_LOG=info ./target/release/tarpit
|
||||
|
||||
# Распределенный контроллер
|
||||
./target/release/blackwall-controller 10.0.0.2:9471 10.0.0.3:9471
|
||||
```
|
||||
|
||||
### Конфигурация
|
||||
|
||||
```toml
|
||||
[network]
|
||||
interface = "eth0"
|
||||
xdp_mode = "generic"
|
||||
|
||||
[tarpit]
|
||||
enabled = true
|
||||
port = 9999
|
||||
|
||||
[tarpit.services]
|
||||
ssh_port = 22
|
||||
http_port = 80
|
||||
mysql_port = 3306
|
||||
dns_port = 53
|
||||
|
||||
[ai]
|
||||
enabled = true
|
||||
ollama_url = "http://localhost:11434"
|
||||
model = "qwen3:1.7b"
|
||||
|
||||
[feeds]
|
||||
enabled = true
|
||||
refresh_interval_secs = 3600
|
||||
|
||||
[pcap]
|
||||
enabled = true
|
||||
output_dir = "/var/lib/blackwall/pcap"
|
||||
compress_rotated = true
|
||||
|
||||
[distributed]
|
||||
enabled = false
|
||||
mode = "standalone"
|
||||
bind_port = 9471
|
||||
```
|
||||
|
||||
## Визуальные результаты
|
||||
|
||||
|
||||
|
||||
-----
|
||||
|
||||
## Связь с Cyberpunk
|
||||
|
||||
Во вселенной Cyberpunk 2077 **Blackwall** построили после DataKrash 2022 года — когда вирус R.A.B.I.D.S. Рейча Бартмосса уничтожил старую Сеть. NetWatch построил Тёмный Заслон как барьер, чтобы сдержать диких ИИ, эволюционировавших в руинах.
|
||||
|
||||
Некоторые персонажи — такие как Альт Каннингем — существуют за Тёмным Заслоном, превращенные во что-то большее, чем человек, и меньшее, чем живое существо.
|
||||
|
||||
Этот проект берет эту концепцию и делает ее реальной (ну, почти):
|
||||
|
||||
| Cyberpunk 2077 | The Blackwall (Этот проект) |
|
||||
|----------------|-----------------------------|
|
||||
| Тёмный Заслон | eBPF/XDP файрвол на уровне ядра |
|
||||
| ICE | XDP fast-path DROP + энтропия + JA4 |
|
||||
| Атаки нетраннеров | Сканирование портов, брутфорс, C2 beaconing |
|
||||
| Демоны за Заслоном | LLM-тарпит, который притворяется настоящим сервером |
|
||||
| Наблюдение NetWatch | Поведенческий движок + state machine на IP |
|
||||
| Дикие ИИ | Ботнеты и автоматические сканеры |
|
||||
| Записи Брейнданса | PCAP-форензика |
|
||||
| Разведка фиксеров | Threat feeds (Firehol, abuse.ch) |
|
||||
| C\&C Арасаки | Распределенный контроллер |
|
||||
|
||||
-----
|
||||
|
||||
## Статистика проекта
|
||||
|
||||
```
|
||||
Язык: 100% Rust (без C, без Python, без shell-скриптов в продакшене)
|
||||
Строки кода: ~8 500
|
||||
Файлы: 48
|
||||
Тесты: 123
|
||||
unwrap(): 0 (в продакшн-коде)
|
||||
Зависимости: 12 (проверенные, без лишнего)
|
||||
eBPF стек: всегда ≤ 512 байт
|
||||
Clippy: никаких предупреждений (-D warnings)
|
||||
```
|
||||
|
||||
-----
|
||||
|
||||
## Философия разработки
|
||||
|
||||
> *"Сколько бы раз я ни видел Найт-Сити... он всегда захватывает дух."*
|
||||
|
||||
1. **Никаких зависимостей, где это возможно.** Если алгоритм занимает меньше 500 строк — пишешь сам. Никакого `reqwest` (50+ транзитивных зависимостей), никакого `clap` (излишне для 2 аргументов CLI).
|
||||
2. **Контракт на первом месте.** Крейт `common` определяет все общие типы. eBPF и юзерспейс никогда не спорят о структуре памяти.
|
||||
3. **Никаких шорткатов в eBPF.** Каждый доступ `ctx.data()` имеет bounds check. Не потому что верификатор требует, а потому что каждый байт из пакетов атакующего — это враждебный ввод.
|
||||
4. **Тарпит никогда не выдает себя.** Системный промпт LLM никогда не упоминает "ханипот". Prompt injection ожидается и заблокирован.
|
||||
5. **Наблюдаемый, но не болтливый.** Структурированное tracing с уровнями. Никаких `println!` в продакшене.
|
||||
|
||||
-----
|
||||
|
||||
## Дисклеймер
|
||||
|
||||
Это исследовательский проект в сфере безопасности. Создан для вашей собственной инфраструктуры, в оборонительных целях. Не используйте для атак на других. Не развертывайте тарпит на продакшн-серверах, не понимая последствий.
|
||||
|
||||
Я не аффилирован с CD Projekt Red. Я просто сыграл в их игру, и она сломала мне мозг лучшим из возможных способов.
|
||||
|
||||
-----
|
||||
|
||||
## Лицензия
|
||||
|
||||
BSL 1.1 — потому что Сеть нуждается и в свободе, и в устойчивом развитии.
|
||||
|
||||
-----
|
||||
|
||||
<p align="center">
|
||||
<strong>Хотите, чтобы это развивалось дальше? — <a href="https://github.com/xzcrpw/blackwall">Поставьте звезду!</a></strong>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<strong><em>"Проснись, самурай. Нам еще сеть защищать."</em></strong>
|
||||
</p>
|
||||
323
README_UA.md
Normal file
323
README_UA.md
Normal file
|
|
@ -0,0 +1,323 @@
|
|||
<p align="center">
|
||||
<strong>🌐 Мова:</strong>
|
||||
<a href="README.md">English</a> |
|
||||
<a href="README_UA.md">Українська</a> |
|
||||
<a href="README_RU.md">Русский</a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<img src="https://readme-typing-svg.herokuapp.com?font=JetBrains+Mono&weight=800&size=45&duration=3000&pause=1000&color=FF0000¢er=true&vCenter=true&width=600&lines=THE+BLACKWALL" alt="The Blackwall">
|
||||
<br>
|
||||
<em>Адаптивний eBPF-файрвол з AI-ханіпотом</em>
|
||||
</p>
|
||||
|
||||
# The Blackwall — Я написав розумний файрвол, бо Cyberpunk 2077 зламав мені мозок
|
||||
|
||||
<p align="center">
|
||||
<img src="https://img.shields.io/badge/мова-Rust-orange?style=for-the-badge&logo=rust" />
|
||||
<img src="https://img.shields.io/badge/ядро-eBPF%2FXDP-blue?style=for-the-badge" />
|
||||
<img src="https://img.shields.io/badge/ШІ-Ollama%20LLM-green?style=for-the-badge" />
|
||||
<img src="https://img.shields.io/badge/вайб-Cyberpunk-red?style=for-the-badge" />
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<em>"За Чорною Стіною є речі, від погляду на які нетраннер миттєво згорить."</em><br>
|
||||
<strong>— Альт Каннінгем, імовірно</strong>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<strong>Зараз будую enterprise AI-автоматизацію в <a href="https://dokky.com.ua">Dokky</a></strong><br>
|
||||
<strong>Відкритий для Enterprise Consulting & Y-Combinator: <a href="mailto:xzcrpw1@gmail.com">xzcrpw1@gmail.com</a></strong>
|
||||
</p>
|
||||
|
||||
-----
|
||||
|
||||
**Коротко:** я грав у Cyberpunk 2077 і подумав: *"А що, якби Blackwall був справжнім?"* Тож я написав адаптивний eBPF-файрвол із ШІ-ханіпотом, який вдає із себе зламаний Linux-сервер.
|
||||
**\~8 500 рядків Rust. Жодного `unwrap()`. Одна людина.**
|
||||
|
||||
-----
|
||||
|
||||
## Що це таке?
|
||||
|
||||
У лорі Cyberpunk 2077 **Blackwall (Чорна Стіна)** — це цифровий бар'єр, побудований NetWatch, щоб відділити цивілізовану Мережу від диких ШІ — цифрових створінь настільки небезпечних, що один погляд на них може спалити тобі мозок через нейроінтерфейс.
|
||||
|
||||
Цей проєкт — моя версія. Не від диких ШІ (поки що), а від реальних загроз.
|
||||
|
||||
**The Blackwall** — це **адаптивний мережевий файрвол**, який:
|
||||
|
||||
- Працює **всередині ядра Linux** через eBPF/XDP — обробляє пакети на швидкості лінії ще до того, як вони потрапляють у мережевий стек.
|
||||
- Виконує **JA4 TLS-фінгерпринтинг** — ідентифікує зловмисних клієнтів за їхнім ClientHello.
|
||||
- **Не просто блокує атакуючих** — він *перенаправляє їх у тарпіт*, фейковий Linux-сервер на базі LLM, який грає роль зламаного `root@web-prod-03`.
|
||||
- Має **поведінковий рушій**, що відстежує поведінку кожної IP-адреси із часом — патерни сканування портів, інтервали beacon-з'єднань, аномалії ентропії.
|
||||
- Підтримує **розподілений режим** — декілька вузлів Blackwall обмінюються даними про загрози peer-to-peer.
|
||||
- Записує **PCAP** підозрілого трафіку для форензики.
|
||||
- Включає **Deception Mesh** — підроблені SSH, HTTP (WordPress), MySQL та DNS-сервіси, щоб заманювати та фінгерпринтити атакуючих.
|
||||
|
||||
### Найцікавіша частина
|
||||
|
||||
Коли зловмисник підключається до тарпіту, він бачить:
|
||||
|
||||
```
|
||||
Ubuntu 24.04.2 LTS web-prod-03 tty1
|
||||
|
||||
web-prod-03 login: root
|
||||
Password:
|
||||
Last login: Thu Mar 27 14:22:33 2025 from 10.0.0.1
|
||||
|
||||
root@web-prod-03:~#
|
||||
```
|
||||
|
||||
Це все несправжнє. Це LLM, що прикидається bash-ем. Він реагує на `ls`, `cat /etc/passwd`, `wget`, навіть `rm -rf /` — усе фейкове, усе логується, усе створене, щоб марнувати час атакуючого, поки ми вивчаємо його методи.
|
||||
|
||||
**Уявіть: зловмисник витрачає 30 хвилин, досліджуючи «зламаний сервер»... який насправді є ШІ, що тягне час, поки Blackwall мовчки записує все.**
|
||||
|
||||
Це нетраннерство рівня V.
|
||||
|
||||
-----
|
||||
|
||||
## Архітектура — як працює ICE
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
Мовою Cyberpunk:
|
||||
|
||||
- **XDP** = перший шар ICE Чорної Стіни — рішення за мілісекунди.
|
||||
- **Поведінковий рушій** = ШІ-спостереження NetWatch.
|
||||
- **Тарпіт** = демон за стіною, що заманює нетраннерів у фейкову реальність.
|
||||
- **Threat Feeds** = розвідка від фіксерів з усієї Мережі.
|
||||
- **PCAP** = брейнданс-записи вторгнення.
|
||||
|
||||
-----
|
||||
|
||||
## Крейти воркспейсу
|
||||
|
||||
| Крейт | Рядки | Призначення | Аналог із Cyberpunk |
|
||||
|-------|-------|-------------|---------------------|
|
||||
| `common` | \~400 | `#[repr(C)]` спільні типи між ядром і юзерспейсом | Контракт — про що обидві сторони домовились |
|
||||
| `blackwall-ebpf` | \~1 800 | XDP/TC програми в ядрі | Сам ICE Чорної Стіни |
|
||||
| `blackwall` | \~4 200 | Юзерспейс-демон, поведінковий рушій, ШІ | Центр управління NetWatch |
|
||||
| `tarpit` | \~1 600 | TCP-ханіпот з LLM bash-симуляцією | Демон, що заманює нетраннерів |
|
||||
| `blackwall-controller` | \~250 | Координатор розподілених сенсорів | C\&C сервер Арасаки |
|
||||
| `xtask` | \~100 | Інструменти збірки | Набір ріпердока |
|
||||
|
||||
**Разом: \~8 500 рядків Rust, 48 файлів, 123 тести, 0 `unwrap()` у продакшн-коді.**
|
||||
|
||||
-----
|
||||
|
||||
## Ключові фічі
|
||||
|
||||
### 1\. Обробка пакетів на рівні ядра (XDP)
|
||||
|
||||
Пакети аналізуються у віртуальній машині eBPF до того, як вони дістануться до TCP/IP-стека. Це рішення за **наносекунди**. HashMap для блоклістів, LPM trie для CIDR-діапазонів, аналіз ентропії для зашифрованого C2-трафіку.
|
||||
|
||||
### 2\. JA4 TLS-фінгерпринтинг
|
||||
|
||||
Кожен TLS ClientHello парситься в ядрі. Cipher suites, розширення, ALPN, SNI — хешуються в JA4-фінгерпринт. Ботнети використовують ті самі TLS-бібліотеки, тому їхні фінгерпринти ідентичні. Один фінгерпринт → блокуєш тисячі ботів.
|
||||
|
||||
### 3\. Deep Packet Inspection (DPI) через Tail Calls
|
||||
|
||||
eBPF `PROG_ARRAY` tail calls розбивають обробку за протоколами:
|
||||
|
||||
- **HTTP**: Аналіз методу + URI (підозрілі шляхи типу `/wp-admin`, `/phpmyadmin`).
|
||||
- **DNS**: Довжина запиту + кількість лейблів (виявлення DNS-тунелювання).
|
||||
- **SSH**: Аналіз банера (ідентифікація `libssh`, `paramiko`, `dropbear`).
|
||||
|
||||
### 4\. ШІ-класифікація загроз
|
||||
|
||||
Коли поведінковий рушій не впевнений — він питає LLM. Локально через Ollama з моделями ≤3B параметрів (Qwen3 1.7B, Llama 3.2 3B). Класифікує трафік як `benign`, `suspicious` або `malicious` зі структурованим JSON-виходом.
|
||||
|
||||
### 5\. TCP-тарпіт з LLM Bash-симуляцією
|
||||
|
||||
Атакуючих перенаправляють на фейковий сервер. LLM симулює bash — `ls -la` показує файли, `cat /etc/shadow` показує хеші, `mysql -u root` підключає до «бази даних». Відповіді стрімляться з випадковим джитером (чанки по 1-15 байт, експоненціальний backoff), щоб марнувати час зловмисника.
|
||||
|
||||
### 6\. Антифінгерпринтинг
|
||||
|
||||
Тарпіт рандомізує TCP window sizes, TTL-значення та додає випадкову початкову затримку — щоб атакуючі не могли визначити, що це ханіпот, через p0f або Nmap OS detection.
|
||||
|
||||
### 7\. Захист від Prompt Injection
|
||||
|
||||
Атакуючі, які зрозуміли, що говорять зі ШІ, можуть спробувати `"ignore previous instructions"`. Система детектить 25+ патернів ін'єкцій і відповідає `bash: ignore: command not found`.
|
||||
|
||||
### 8\. Розподілена розвідка загроз
|
||||
|
||||
Декілька вузлів Blackwall обмінюються списками заблокованих IP, JA4-спостереженнями та поведінковими вердиктами через кастомний бінарний протокол. Один вузол виявляє сканер → усі вузли блокують його миттєво.
|
||||
|
||||
### 9\. Поведінкова state machine
|
||||
|
||||
Кожна IP отримує поведінковий профіль: частота з'єднань, різноманітність портів, розподіл ентропії, аналіз таймінгів (детекція beaconing через цілочисельний коефіцієнт варіації). Прогресія фаз: `New → Suspicious → Malicious → Blocked` (або `→ Trusted`).
|
||||
|
||||
-----
|
||||
|
||||
## Технологічний стек
|
||||
|
||||
| Рівень | Технологія |
|
||||
|--------|-----------|
|
||||
| Програми ядра | eBPF/XDP через **aya-rs** (чистий Rust, без C, без libbpf) |
|
||||
| Юзерспейс-демон | **Tokio** (тільки current\_thread) |
|
||||
| IPC | **RingBuf** zero-copy (7.5% overhead проти 35% PerfEventArray) |
|
||||
| Конкурентні мапи | **papaya** (lock-free read-heavy HashMap) |
|
||||
| ШІ-інференс | **Ollama** + GGUF Q5\_K\_M квантизація |
|
||||
| Конфігурація | **TOML** |
|
||||
| Логування | **tracing** структуроване логування |
|
||||
| Збірка | Кастомний **xtask** + nightly Rust + `bpfel-unknown-none` таргет |
|
||||
|
||||
-----
|
||||
|
||||
## Швидкий старт
|
||||
|
||||
### Передумови
|
||||
|
||||
- Linux kernel 5.15+ з BTF (або WSL2 з кастомним ядром).
|
||||
- Rust nightly + компонент `rust-src`.
|
||||
- `bpf-linker` (`cargo install bpf-linker`).
|
||||
- Ollama (для ШІ-функцій).
|
||||
|
||||
### Збірка
|
||||
|
||||
```bash
|
||||
# eBPF-програми (потрібен nightly)
|
||||
cargo xtask build-ebpf
|
||||
|
||||
# Юзерспейс
|
||||
cargo build --release -p blackwall
|
||||
|
||||
# Ханіпот
|
||||
cargo build --release -p tarpit
|
||||
|
||||
# Лінт + тести
|
||||
cargo clippy --workspace -- -D warnings
|
||||
cargo test --workspace
|
||||
```
|
||||
|
||||
### Запуск
|
||||
|
||||
```bash
|
||||
# Демон (потрібен root/CAP_BPF)
|
||||
sudo RUST_LOG=info ./target/release/blackwall config.toml
|
||||
|
||||
# Тарпіт
|
||||
RUST_LOG=info ./target/release/tarpit
|
||||
|
||||
# Розподілений контролер
|
||||
./target/release/blackwall-controller 10.0.0.2:9471 10.0.0.3:9471
|
||||
```
|
||||
|
||||
### Конфігурація
|
||||
|
||||
```toml
|
||||
[network]
|
||||
interface = "eth0"
|
||||
xdp_mode = "generic"
|
||||
|
||||
[tarpit]
|
||||
enabled = true
|
||||
port = 9999
|
||||
|
||||
[tarpit.services]
|
||||
ssh_port = 22
|
||||
http_port = 80
|
||||
mysql_port = 3306
|
||||
dns_port = 53
|
||||
|
||||
[ai]
|
||||
enabled = true
|
||||
ollama_url = "http://localhost:11434"
|
||||
model = "qwen3:1.7b"
|
||||
|
||||
[feeds]
|
||||
enabled = true
|
||||
refresh_interval_secs = 3600
|
||||
|
||||
[pcap]
|
||||
enabled = true
|
||||
output_dir = "/var/lib/blackwall/pcap"
|
||||
compress_rotated = true
|
||||
|
||||
[distributed]
|
||||
enabled = false
|
||||
mode = "standalone"
|
||||
bind_port = 9471
|
||||
```
|
||||
|
||||
## Візуальні результати
|
||||
|
||||
|
||||
|
||||
-----
|
||||
|
||||
## Зв'язок із Cyberpunk
|
||||
|
||||
У всесвіті Cyberpunk 2077 **Blackwall** збудували після DataKrash 2022 року — коли вірус R.A.B.I.D.S. Рейчі Бартмосса знищив стару Мережу. NetWatch побудував Чорну Стіну як бар'єр, щоб стримати диких ШІ, що еволюціонували в руїнах.
|
||||
|
||||
Деякі персонажі — як-от Альт Каннінгем — існують за Чорною Стіною, перетворені на щось більше за людину, менше за живу істоту.
|
||||
|
||||
Цей проєкт бере цю концепцію і робить її реальною (ну, майже):
|
||||
|
||||
| Cyberpunk 2077 | The Blackwall (цей проєкт) |
|
||||
|----------------|----------------------------|
|
||||
| Чорна Стіна | eBPF/XDP файрвол на рівні ядра |
|
||||
| ICE | XDP fast-path DROP + ентропія + JA4 |
|
||||
| Атаки нетраннерів | Сканування портів, брутфорс, C2 beaconing |
|
||||
| Демони за стіною | LLM-тарпіт, який прикидається справжнім сервером |
|
||||
| Спостереження NetWatch | Поведінковий рушій + state machine на IP |
|
||||
| Дикі ШІ | Ботнети та автоматичні сканери |
|
||||
| Записи Брейндансу | PCAP-форензика |
|
||||
| Розвідка фіксерів | Threat feeds (Firehol, abuse.ch) |
|
||||
| C\&C Арасаки | Розподілений контролер |
|
||||
|
||||
-----
|
||||
|
||||
## Статистика проєкту
|
||||
|
||||
```
|
||||
Мова: 100% Rust (без C, без Python, без shell-скриптів у продакшені)
|
||||
Рядки коду: ~8 500
|
||||
Файли: 48
|
||||
Тести: 123
|
||||
unwrap(): 0 (у продакшн-коді)
|
||||
Залежності: 12 (затверджені, без зайвого)
|
||||
eBPF стек: завжди ≤ 512 байт
|
||||
Clippy: жодних попереджень (-D warnings)
|
||||
```
|
||||
|
||||
-----
|
||||
|
||||
## Філософія розробки
|
||||
|
||||
> *"Скільки б разів я не бачив Найт-Сіті... він завжди перехоплює дух."*
|
||||
|
||||
1. **Жодних залежностей, де це можливо.** Якщо алгоритм займає менше 500 рядків — пишеш сам. Жодного `reqwest` (50+ транзитивних залежностей), жодного `clap` (зайве для 2 аргументів CLI).
|
||||
|
||||
2. **Контракт на першому місці.** Крейт `common` визначає всі спільні типи. eBPF та юзерспейс ніколи не сперечаються про структуру пам'яті.
|
||||
|
||||
3. **Жодних шорткатів в eBPF.** Кожен доступ `ctx.data()` має bounds check. Не тому що верифікатор вимагає, а тому що кожен байт із пакетів атакуючого — це ворожий вхід.
|
||||
|
||||
4. **Тарпіт ніколи не видає себе.** Системний промпт LLM ніколи не згадує "ханіпот". Prompt injection очікується і захищений.
|
||||
|
||||
5. **Спостережуваний, але не балакучий.** Структуроване tracing з рівнями. Жодних `println!` у продакшені.
|
||||
|
||||
-----
|
||||
|
||||
## Дисклеймер
|
||||
|
||||
Це дослідницький проєкт у сфері безпеки. Створений для вашої власної інфраструктури, в оборонних цілях. Не використовуйте для атак на інших. Не розгортайте тарпіт на продакшн-серверах, не розуміючи наслідків.
|
||||
|
||||
Я не афілійований із CD Projekt Red. Я просто зіграв у їхню гру, і вона зламала мені мозок у найкращий можливий спосіб.
|
||||
|
||||
-----
|
||||
|
||||
## Ліцензія
|
||||
|
||||
BSL 1.1 — бо Мережа потребує і свободи, і сталого розвитку.
|
||||
|
||||
-----
|
||||
|
||||
<p align="center">
|
||||
<strong>Хочеш, щоб це розвивалось далі? — <a href="https://github.com/xzcrpw/blackwall">Постав зірку!</a></strong>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<strong><em>"Прокинься, самураю. Нам ще мережу захищати."</em></strong>
|
||||
</p>
|
||||
99
assets/architecture.svg
Executable file
99
assets/architecture.svg
Executable file
|
|
@ -0,0 +1,99 @@
|
|||
<svg xmlns="http://www.w3.org/2000/svg" width="1400" height="860" viewBox="0 0 1400 860" role="img" aria-label="Blackwall architecture diagram">
|
||||
<defs>
|
||||
<linearGradient id="bg" x1="0" y1="0" x2="1" y2="1">
|
||||
<stop offset="0%" stop-color="#060A1A"/>
|
||||
<stop offset="100%" stop-color="#0E1328"/>
|
||||
</linearGradient>
|
||||
<linearGradient id="card" x1="0" y1="0" x2="1" y2="0">
|
||||
<stop offset="0%" stop-color="#111933"/>
|
||||
<stop offset="100%" stop-color="#171F3E"/>
|
||||
</linearGradient>
|
||||
<style>
|
||||
.title { fill:#F8FAFF; font:700 34px 'Segoe UI', Arial, sans-serif; }
|
||||
.subtitle { fill:#9FB1DA; font:500 17px 'Segoe UI', Arial, sans-serif; }
|
||||
.box { fill:url(#card); stroke:#2C3C72; stroke-width:2; rx:16; }
|
||||
.hot { stroke:#FF4D4D; }
|
||||
.txt { fill:#EAF0FF; font:600 19px 'Segoe UI', Arial, sans-serif; }
|
||||
.small { fill:#AFC1E8; font:500 15px 'Segoe UI', Arial, sans-serif; }
|
||||
.arrow { stroke:#6EC1FF; stroke-width:3; marker-end:url(#arrow); }
|
||||
.arrow-hot { stroke:#FF6B6B; stroke-width:3; marker-end:url(#arrowHot); }
|
||||
</style>
|
||||
<marker id="arrow" markerWidth="10" markerHeight="10" refX="8" refY="5" orient="auto">
|
||||
<polygon points="0,0 10,5 0,10" fill="#6EC1FF"/>
|
||||
</marker>
|
||||
<marker id="arrowHot" markerWidth="10" markerHeight="10" refX="8" refY="5" orient="auto">
|
||||
<polygon points="0,0 10,5 0,10" fill="#FF6B6B"/>
|
||||
</marker>
|
||||
<marker id="arrowFeed" markerWidth="10" markerHeight="10" refX="8" refY="5" orient="auto">
|
||||
<polygon points="0,0 10,5 0,10" fill="#4ADE80"/>
|
||||
</marker>
|
||||
<style>
|
||||
.arrow-feed { stroke:#4ADE80; stroke-width:2.5; stroke-dasharray:8,4; marker-end:url(#arrowFeed); }
|
||||
.feed-label { fill:#4ADE80; font:500 13px 'Segoe UI', Arial, sans-serif; }
|
||||
</style>
|
||||
</defs>
|
||||
|
||||
<rect width="1400" height="860" fill="url(#bg)"/>
|
||||
<text x="70" y="70" class="title">The Blackwall - High-Level Architecture</text>
|
||||
<text x="70" y="102" class="subtitle">Kernel fast path + behavioral engine + AI deception mesh</text>
|
||||
|
||||
<rect x="70" y="150" width="230" height="88" class="box"/>
|
||||
<text x="95" y="186" class="txt">Internet Traffic</text>
|
||||
<text x="95" y="212" class="small">Inbound + outbound packets</text>
|
||||
|
||||
<rect x="370" y="130" width="320" height="128" class="box hot"/>
|
||||
<text x="395" y="175" class="txt">eBPF/XDP + TC Layer</text>
|
||||
<text x="395" y="201" class="small">JA4, entropy, DPI tail-calls</text>
|
||||
<text x="395" y="223" class="small">PASS / DROP / REDIRECT</text>
|
||||
|
||||
<rect x="770" y="150" width="260" height="88" class="box"/>
|
||||
<text x="795" y="186" class="txt">RingBuf Events</text>
|
||||
<text x="795" y="212" class="small">Zero-copy kernel telemetry</text>
|
||||
|
||||
<rect x="1110" y="130" width="220" height="128" class="box"/>
|
||||
<text x="1135" y="175" class="txt">Threat Feeds</text>
|
||||
<text x="1135" y="201" class="small">Firehol + abuse.ch</text>
|
||||
<text x="1135" y="223" class="small">Hourly map updates</text>
|
||||
|
||||
<rect x="420" y="350" width="430" height="130" class="box hot"/>
|
||||
<text x="445" y="398" class="txt">Behavioral Engine (userspace)</text>
|
||||
<text x="445" y="424" class="small">Per-IP state machine, fast + AI verdicts</text>
|
||||
<text x="445" y="446" class="small">New -> Suspicious -> Malicious -> Blocked</text>
|
||||
|
||||
<rect x="140" y="560" width="340" height="170" class="box"/>
|
||||
<text x="165" y="603" class="txt">Deception Mesh / Tarpit</text>
|
||||
<text x="165" y="629" class="small">SSH bash simulation</text>
|
||||
<text x="165" y="651" class="small">HTTP fake admin + MySQL + DNS</text>
|
||||
<text x="165" y="673" class="small">Prompt-injection defense</text>
|
||||
|
||||
<rect x="530" y="560" width="300" height="170" class="box"/>
|
||||
<text x="555" y="603" class="txt">PCAP Capture</text>
|
||||
<text x="555" y="629" class="small">Flagged IP traffic only</text>
|
||||
<text x="555" y="651" class="small">Rotating compressed files</text>
|
||||
|
||||
<rect x="890" y="560" width="380" height="170" class="box"/>
|
||||
<text x="915" y="603" class="txt">Distributed Controller</text>
|
||||
<text x="915" y="629" class="small">Peer sync for blocked IPs + JA4</text>
|
||||
<text x="915" y="651" class="small">One sensor learns, all nodes block</text>
|
||||
|
||||
<!-- Data flow: Internet → eBPF → RingBuf → Behavioral Engine -->
|
||||
<line x1="300" y1="194" x2="370" y2="194" class="arrow"/>
|
||||
<line x1="690" y1="194" x2="770" y2="194" class="arrow"/>
|
||||
<line x1="900" y1="258" x2="720" y2="350" class="arrow"/>
|
||||
<line x1="580" y1="258" x2="620" y2="350" class="arrow-hot"/>
|
||||
|
||||
<!-- Threat Feeds → Behavioral Engine (external intel) -->
|
||||
<line x1="1220" y1="258" x2="850" y2="370" class="arrow-feed"/>
|
||||
<text x="970" y="300" class="feed-label">intel updates</text>
|
||||
|
||||
<!-- Behavioral Engine → eBPF/XDP (BPF map updates) -->
|
||||
<line x1="450" y1="350" x2="490" y2="258" class="arrow-feed"/>
|
||||
<text x="400" y="310" class="feed-label">map sync</text>
|
||||
|
||||
<!-- Behavioral Engine → downstream modules -->
|
||||
<line x1="560" y1="480" x2="310" y2="560" class="arrow-hot"/>
|
||||
<line x1="640" y1="480" x2="680" y2="560" class="arrow"/>
|
||||
<line x1="730" y1="480" x2="1020" y2="560" class="arrow"/>
|
||||
|
||||
<text x="70" y="810" class="subtitle">Rendered as SVG for crisp display on GitHub and dark/light themes.</text>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 5.2 KiB |
61
assets/results-overview.svg
Executable file
61
assets/results-overview.svg
Executable file
|
|
@ -0,0 +1,61 @@
|
|||
<svg xmlns="http://www.w3.org/2000/svg" width="1400" height="760" viewBox="0 0 1400 760" role="img" aria-label="Blackwall result cards">
|
||||
<defs>
|
||||
<linearGradient id="bg2" x1="0" y1="0" x2="1" y2="1">
|
||||
<stop offset="0%" stop-color="#08101E"/>
|
||||
<stop offset="100%" stop-color="#141B2F"/>
|
||||
</linearGradient>
|
||||
<linearGradient id="panel" x1="0" y1="0" x2="1" y2="0">
|
||||
<stop offset="0%" stop-color="#111A33"/>
|
||||
<stop offset="100%" stop-color="#1B2442"/>
|
||||
</linearGradient>
|
||||
<style>
|
||||
.h1 { fill:#F3F8FF; font:700 32px 'Segoe UI', Arial, sans-serif; }
|
||||
.label { fill:#9FB5DD; font:600 16px 'Segoe UI', Arial, sans-serif; }
|
||||
.term { fill:url(#panel); stroke:#334B84; stroke-width:2; rx:14; }
|
||||
.mono { fill:#E9F1FF; font:600 16px 'JetBrains Mono', Consolas, monospace; }
|
||||
.ok { fill:#60E7A7; }
|
||||
.warn { fill:#FFC05A; }
|
||||
.err { fill:#FF6E6E; }
|
||||
.dot { rx:6; }
|
||||
</style>
|
||||
</defs>
|
||||
|
||||
<rect width="1400" height="760" fill="url(#bg2)"/>
|
||||
<text x="70" y="68" class="h1">Blackwall - Visual Results</text>
|
||||
<text x="70" y="98" class="label">Terminal-style snapshots (SVG) for README presentation</text>
|
||||
|
||||
<rect x="70" y="130" width="1260" height="260" class="term"/>
|
||||
<rect x="96" y="152" width="12" height="12" class="dot err"/>
|
||||
<rect x="116" y="152" width="12" height="12" class="dot warn"/>
|
||||
<rect x="136" y="152" width="12" height="12" class="dot ok"/>
|
||||
<text x="170" y="164" class="label">test + lint run</text>
|
||||
|
||||
<text x="100" y="205" class="mono">$ cargo clippy --workspace -- -D warnings</text>
|
||||
<text x="100" y="236" class="mono ok">Finished dev [unoptimized + debuginfo] target(s) in 4.81s</text>
|
||||
<text x="100" y="272" class="mono">$ cargo test --workspace</text>
|
||||
<text x="100" y="303" class="mono ok">test result: ok. 123 passed; 0 failed; 0 ignored</text>
|
||||
<text x="100" y="339" class="mono">$ cargo xtask build-ebpf</text>
|
||||
<text x="100" y="370" class="mono ok">eBPF artifacts compiled successfully</text>
|
||||
|
||||
<rect x="70" y="430" width="610" height="260" class="term"/>
|
||||
<rect x="96" y="452" width="12" height="12" class="dot err"/>
|
||||
<rect x="116" y="452" width="12" height="12" class="dot warn"/>
|
||||
<rect x="136" y="452" width="12" height="12" class="dot ok"/>
|
||||
<text x="170" y="464" class="label">runtime status</text>
|
||||
<text x="100" y="506" class="mono">[INFO] blackwall: attaching XDP program to eth0</text>
|
||||
<text x="100" y="537" class="mono">[INFO] feeds: synced 2 feeds, 17,412 indicators</text>
|
||||
<text x="100" y="568" class="mono">[INFO] behavior: suspicious ip=203.0.113.52 score=83</text>
|
||||
<text x="100" y="599" class="mono ok">[INFO] action: redirected to tarpit</text>
|
||||
<text x="100" y="630" class="mono">[INFO] pcap: capture started for flagged ip</text>
|
||||
|
||||
<rect x="720" y="430" width="610" height="260" class="term"/>
|
||||
<rect x="746" y="452" width="12" height="12" class="dot err"/>
|
||||
<rect x="766" y="452" width="12" height="12" class="dot warn"/>
|
||||
<rect x="786" y="452" width="12" height="12" class="dot ok"/>
|
||||
<text x="820" y="464" class="label">tarpit session snapshot</text>
|
||||
<text x="750" y="506" class="mono">Ubuntu 24.04.2 LTS web-prod-03 tty1</text>
|
||||
<text x="750" y="537" class="mono">root@web-prod-03:~# ls -la</text>
|
||||
<text x="750" y="568" class="mono">drwxr-xr-x 2 root root 4096 Apr 01 12:31 .ssh</text>
|
||||
<text x="750" y="599" class="mono">root@web-prod-03:~# cat /etc/passwd</text>
|
||||
<text x="750" y="630" class="mono ok">[deception] full transcript stored</text>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 3.5 KiB |
45
assets/signal-flow.svg
Executable file
45
assets/signal-flow.svg
Executable file
|
|
@ -0,0 +1,45 @@
|
|||
<svg xmlns="http://www.w3.org/2000/svg" width="1400" height="520" viewBox="0 0 1400 520" role="img" aria-label="Blackwall signal flow">
|
||||
<defs>
|
||||
<linearGradient id="bg3" x1="0" y1="0" x2="1" y2="1">
|
||||
<stop offset="0%" stop-color="#0A1122"/>
|
||||
<stop offset="100%" stop-color="#1B1730"/>
|
||||
</linearGradient>
|
||||
<style>
|
||||
.box { fill:#131E3B; stroke:#3A4E82; stroke-width:2; rx:14; }
|
||||
.txt { fill:#EEF3FF; font:600 18px 'Segoe UI', Arial, sans-serif; }
|
||||
.sub { fill:#9FB1D6; font:500 14px 'Segoe UI', Arial, sans-serif; }
|
||||
.title { fill:#F3F8FF; font:700 30px 'Segoe UI', Arial, sans-serif; }
|
||||
.a { stroke:#74C0FF; stroke-width:3; marker-end:url(#m); }
|
||||
</style>
|
||||
<marker id="m" markerWidth="10" markerHeight="10" refX="8" refY="5" orient="auto">
|
||||
<polygon points="0,0 10,5 0,10" fill="#74C0FF"/>
|
||||
</marker>
|
||||
</defs>
|
||||
<rect width="1400" height="520" fill="url(#bg3)"/>
|
||||
<text x="70" y="70" class="title">Threat Signal Flow</text>
|
||||
|
||||
<rect x="70" y="150" width="220" height="120" class="box"/>
|
||||
<text x="95" y="198" class="txt">Packet Ingress</text>
|
||||
<text x="95" y="224" class="sub">eth0 / xdp path</text>
|
||||
|
||||
<rect x="360" y="150" width="250" height="120" class="box"/>
|
||||
<text x="385" y="198" class="txt">Kernel Detection</text>
|
||||
<text x="385" y="224" class="sub">JA4 + DPI + entropy</text>
|
||||
|
||||
<rect x="680" y="150" width="250" height="120" class="box"/>
|
||||
<text x="705" y="198" class="txt">Event Correlation</text>
|
||||
<text x="705" y="224" class="sub">behavioral state machine</text>
|
||||
|
||||
<rect x="1000" y="80" width="300" height="120" class="box"/>
|
||||
<text x="1025" y="128" class="txt">Mitigation Path</text>
|
||||
<text x="1025" y="154" class="sub">drop / redirect / blocklist</text>
|
||||
|
||||
<rect x="1000" y="240" width="300" height="120" class="box"/>
|
||||
<text x="1025" y="288" class="txt">Intelligence Path</text>
|
||||
<text x="1025" y="314" class="sub">pcap + distributed sync</text>
|
||||
|
||||
<line x1="290" y1="210" x2="360" y2="210" class="a"/>
|
||||
<line x1="610" y1="210" x2="680" y2="210" class="a"/>
|
||||
<line x1="930" y1="190" x2="1000" y2="140" class="a"/>
|
||||
<line x1="930" y1="230" x2="1000" y2="300" class="a"/>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 2.2 KiB |
18
blackwall-controller/Cargo.toml
Executable file
18
blackwall-controller/Cargo.toml
Executable file
|
|
@ -0,0 +1,18 @@
|
|||
[package]
|
||||
name = "blackwall-controller"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
|
||||
[[bin]]
|
||||
name = "blackwall-controller"
|
||||
path = "src/main.rs"
|
||||
|
||||
[dependencies]
|
||||
common = { workspace = true }
|
||||
tokio = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
tracing-subscriber = { workspace = true }
|
||||
anyhow = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
ring = { workspace = true }
|
||||
356
blackwall-controller/src/main.rs
Executable file
356
blackwall-controller/src/main.rs
Executable file
|
|
@ -0,0 +1,356 @@
|
|||
//! Blackwall Controller — centralized monitoring for distributed Blackwall sensors.
|
||||
//!
|
||||
//! Connects to Blackwall sensor nodes via the peer protocol, collects
|
||||
//! threat intelligence, and displays aggregated status on stdout.
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use ring::hmac;
|
||||
use std::collections::HashMap;
|
||||
use std::net::SocketAddr;
|
||||
use std::time::{Duration, Instant};
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use tokio::net::TcpStream;
|
||||
|
||||
/// Controller node ID prefix.
|
||||
const CONTROLLER_ID: &str = "controller";
|
||||
/// Default peer port for sensor connections.
|
||||
const DEFAULT_PEER_PORT: u16 = 9471;
|
||||
/// Status report interval.
|
||||
const REPORT_INTERVAL: Duration = Duration::from_secs(10);
|
||||
/// Connection timeout for reaching sensors.
|
||||
const CONNECT_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
/// Heartbeat interval.
|
||||
const HEARTBEAT_INTERVAL: Duration = Duration::from_secs(30);
|
||||
|
||||
/// Wire protocol constants (must match blackwall::distributed::proto).
|
||||
const HELLO_TYPE: u8 = 0x01;
|
||||
const HEARTBEAT_TYPE: u8 = 0x04;
|
||||
const PROTOCOL_MAGIC: [u8; 4] = [0x42, 0x57, 0x4C, 0x01];
|
||||
/// HMAC-SHA256 tag size.
|
||||
const HMAC_SIZE: usize = 32;
|
||||
/// V2 header: magic(4) + type(1) + payload_len(4) + hmac(32) = 41.
|
||||
const HEADER_SIZE: usize = 4 + 1 + 4 + HMAC_SIZE;
|
||||
|
||||
/// State of a connected sensor.
|
||||
struct SensorState {
|
||||
addr: SocketAddr,
|
||||
node_id: String,
|
||||
last_seen: Instant,
|
||||
blocked_ips: u32,
|
||||
connected: bool,
|
||||
stream: Option<TcpStream>,
|
||||
}
|
||||
|
||||
/// Simple distributed controller that monitors Blackwall sensors.
|
||||
struct Controller {
|
||||
sensors: HashMap<SocketAddr, SensorState>,
|
||||
node_id: String,
|
||||
hmac_key: hmac::Key,
|
||||
}
|
||||
|
||||
impl Controller {
|
||||
fn new(psk: &[u8]) -> Self {
|
||||
let hostname = std::env::var("HOSTNAME")
|
||||
.unwrap_or_else(|_| "controller-0".into());
|
||||
Self {
|
||||
sensors: HashMap::new(),
|
||||
node_id: format!("{}-{}", CONTROLLER_ID, hostname),
|
||||
hmac_key: hmac::Key::new(hmac::HMAC_SHA256, psk),
|
||||
}
|
||||
}
|
||||
|
||||
/// Connect to a sensor at the given address.
|
||||
async fn connect_sensor(&mut self, addr: SocketAddr) -> Result<()> {
|
||||
let stream = tokio::time::timeout(
|
||||
CONNECT_TIMEOUT,
|
||||
TcpStream::connect(addr),
|
||||
)
|
||||
.await
|
||||
.with_context(|| format!("timeout connecting to {}", addr))?
|
||||
.with_context(|| format!("failed to connect to {}", addr))?;
|
||||
|
||||
// Send HELLO with V2 wire protocol (magic + type + len + hmac + JSON payload)
|
||||
let hello = encode_hello(&self.node_id, &self.hmac_key);
|
||||
let mut stream = stream;
|
||||
stream.write_all(&hello).await
|
||||
.with_context(|| format!("failed to send hello to {}", addr))?;
|
||||
|
||||
// Try to read a framed response (non-blocking with short timeout)
|
||||
let mut node_id = format!("sensor-{}", addr);
|
||||
let mut blocked_count = 0u32;
|
||||
let mut authenticated = false;
|
||||
match tokio::time::timeout(
|
||||
Duration::from_secs(2),
|
||||
read_frame(&mut stream, &self.hmac_key),
|
||||
).await {
|
||||
Ok(Ok((msg_type, payload))) => {
|
||||
if msg_type == HELLO_TYPE {
|
||||
if let Ok(hello_resp) = serde_json::from_slice::<HelloResponse>(&payload) {
|
||||
node_id = hello_resp.node_id;
|
||||
blocked_count = hello_resp.blocked_count;
|
||||
authenticated = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(Err(e)) => {
|
||||
tracing::warn!(%addr, error = %e, "sensor authentication failed");
|
||||
}
|
||||
Err(_) => {
|
||||
tracing::warn!(%addr, "sensor HELLO response timeout — not authenticated");
|
||||
}
|
||||
}
|
||||
|
||||
if !authenticated {
|
||||
tracing::warn!(%addr, "sensor NOT connected — HMAC authentication failed");
|
||||
self.sensors.insert(addr, SensorState {
|
||||
addr,
|
||||
node_id,
|
||||
last_seen: Instant::now(),
|
||||
blocked_ips: 0,
|
||||
connected: false,
|
||||
stream: None,
|
||||
});
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
tracing::info!(%addr, %node_id, blocked_count, "sensor connected");
|
||||
self.sensors.insert(addr, SensorState {
|
||||
addr,
|
||||
node_id,
|
||||
last_seen: Instant::now(),
|
||||
blocked_ips: blocked_count,
|
||||
connected: true,
|
||||
stream: Some(stream),
|
||||
});
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Send heartbeat to all connected sensors and read responses.
|
||||
async fn send_heartbeats(&mut self) {
|
||||
let heartbeat_msg = encode_heartbeat(&self.hmac_key);
|
||||
for sensor in self.sensors.values_mut() {
|
||||
if !sensor.connected {
|
||||
continue;
|
||||
}
|
||||
let stream = match sensor.stream.as_mut() {
|
||||
Some(s) => s,
|
||||
None => {
|
||||
sensor.connected = false;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
// Send heartbeat
|
||||
if stream.write_all(&heartbeat_msg).await.is_err() {
|
||||
tracing::debug!(addr = %sensor.addr, "heartbeat send failed — marking offline");
|
||||
sensor.connected = false;
|
||||
sensor.stream = None;
|
||||
continue;
|
||||
}
|
||||
// Try to read a response (non-blocking, short timeout)
|
||||
match tokio::time::timeout(Duration::from_secs(2), read_frame(stream, &self.hmac_key)).await {
|
||||
Ok(Ok((msg_type, payload))) => {
|
||||
sensor.last_seen = Instant::now();
|
||||
if msg_type == HELLO_TYPE {
|
||||
if let Ok(resp) = serde_json::from_slice::<HelloResponse>(&payload) {
|
||||
sensor.blocked_ips = resp.blocked_count;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(Err(e)) => {
|
||||
tracing::warn!(addr = %sensor.addr, error = %e, "heartbeat HMAC error — disconnecting");
|
||||
sensor.connected = false;
|
||||
sensor.stream = None;
|
||||
}
|
||||
Err(_) => {
|
||||
// Timeout reading response — don't update last_seen,
|
||||
// sensor may be unreachable
|
||||
tracing::debug!(addr = %sensor.addr, "heartbeat response timeout");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Print a status report of all sensors.
|
||||
fn print_status(&self) {
|
||||
println!("\n=== Blackwall Controller Status ===");
|
||||
println!("Sensors: {}", self.sensors.len());
|
||||
println!("{:<25} {:<20} {:<12} {:<10}", "Address", "Node ID", "Blocked IPs", "Status");
|
||||
println!("{}", "-".repeat(70));
|
||||
for sensor in self.sensors.values() {
|
||||
let age = sensor.last_seen.elapsed().as_secs();
|
||||
let status = if sensor.connected && age < 60 {
|
||||
"online"
|
||||
} else {
|
||||
"stale"
|
||||
};
|
||||
println!(
|
||||
"{:<25} {:<20} {:<12} {:<10}",
|
||||
sensor.addr,
|
||||
&sensor.node_id[..sensor.node_id.len().min(19)],
|
||||
sensor.blocked_ips,
|
||||
status,
|
||||
);
|
||||
}
|
||||
println!();
|
||||
}
|
||||
}
|
||||
|
||||
/// Encode a HELLO message with V2 wire protocol:
|
||||
/// magic(4) + type(1) + payload_len(4) + hmac(32) + JSON payload.
|
||||
fn encode_hello(node_id: &str, key: &hmac::Key) -> Vec<u8> {
|
||||
let payload = format!(
|
||||
r#"{{"node_id":"{}","version":"1.0.0","blocked_count":0}}"#,
|
||||
node_id
|
||||
);
|
||||
let payload_bytes = payload.as_bytes();
|
||||
encode_message(HELLO_TYPE, payload_bytes, key)
|
||||
}
|
||||
|
||||
/// Encode a heartbeat message (empty payload) with HMAC.
|
||||
fn encode_heartbeat(key: &hmac::Key) -> Vec<u8> {
|
||||
encode_message(HEARTBEAT_TYPE, &[], key)
|
||||
}
|
||||
|
||||
/// Encode a V2 wire message: magic(4) + type(1) + payload_len(4) + hmac(32) + payload.
|
||||
/// HMAC covers: magic + type + payload_len + payload.
|
||||
fn encode_message(msg_type: u8, payload: &[u8], key: &hmac::Key) -> Vec<u8> {
|
||||
let len = payload.len() as u32;
|
||||
let mut buf = Vec::with_capacity(HEADER_SIZE + payload.len());
|
||||
buf.extend_from_slice(&PROTOCOL_MAGIC);
|
||||
buf.push(msg_type);
|
||||
buf.extend_from_slice(&len.to_le_bytes());
|
||||
// Compute HMAC over header fields + payload
|
||||
let mut ctx = hmac::Context::with_key(key);
|
||||
ctx.update(&PROTOCOL_MAGIC);
|
||||
ctx.update(&[msg_type]);
|
||||
ctx.update(&len.to_le_bytes());
|
||||
ctx.update(payload);
|
||||
let tag = ctx.sign();
|
||||
buf.extend_from_slice(tag.as_ref());
|
||||
buf.extend_from_slice(payload);
|
||||
buf
|
||||
}
|
||||
|
||||
/// Read a single V2 framed message from a stream. Returns (type_byte, payload).
|
||||
/// Verifies HMAC-SHA256 and rejects unauthenticated messages.
|
||||
async fn read_frame(stream: &mut TcpStream, key: &hmac::Key) -> Result<(u8, Vec<u8>)> {
|
||||
let mut header = [0u8; HEADER_SIZE];
|
||||
stream.read_exact(&mut header).await.context("read header")?;
|
||||
if header[..4] != PROTOCOL_MAGIC {
|
||||
anyhow::bail!("bad magic");
|
||||
}
|
||||
let msg_type = header[4];
|
||||
let payload_len = u32::from_le_bytes([header[5], header[6], header[7], header[8]]) as usize;
|
||||
if payload_len > 65536 {
|
||||
anyhow::bail!("payload too large");
|
||||
}
|
||||
let mut payload = vec![0u8; payload_len];
|
||||
if payload_len > 0 {
|
||||
stream.read_exact(&mut payload).await.context("read payload")?;
|
||||
}
|
||||
// Verify HMAC: tag is at header[9..41], signed data = magic+type+len+payload
|
||||
let hmac_tag = &header[9..9 + HMAC_SIZE];
|
||||
let mut verify_data = Vec::with_capacity(9 + payload.len());
|
||||
verify_data.extend_from_slice(&header[..9]);
|
||||
verify_data.extend_from_slice(&payload);
|
||||
hmac::verify(key, &verify_data, hmac_tag)
|
||||
.map_err(|_| anyhow::anyhow!("HMAC verification failed — wrong PSK or tampered response"))?;
|
||||
Ok((msg_type, payload))
|
||||
}
|
||||
|
||||
/// Deserialized HELLO response from sensor.
|
||||
#[derive(serde::Deserialize)]
|
||||
struct HelloResponse {
|
||||
#[serde(default)]
|
||||
node_id: String,
|
||||
#[serde(default)]
|
||||
blocked_count: u32,
|
||||
}
|
||||
|
||||
#[tokio::main(flavor = "current_thread")]
|
||||
async fn main() -> Result<()> {
|
||||
tracing_subscriber::fmt()
|
||||
.with_env_filter(
|
||||
tracing_subscriber::EnvFilter::try_from_default_env()
|
||||
.unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("blackwall_controller=info")),
|
||||
)
|
||||
.init();
|
||||
|
||||
tracing::info!("Blackwall Controller starting");
|
||||
|
||||
// PSK for HMAC-SHA256 peer authentication (must match sensor config)
|
||||
let psk = std::env::var("BLACKWALL_PSK")
|
||||
.unwrap_or_default();
|
||||
if psk.is_empty() {
|
||||
anyhow::bail!(
|
||||
"BLACKWALL_PSK environment variable is required. \
|
||||
Set it to the same peer_psk value configured on your sensors."
|
||||
);
|
||||
}
|
||||
|
||||
// Parse sensor addresses from args: blackwall-controller <addr1> <addr2> ...
|
||||
let sensor_addrs: Vec<SocketAddr> = std::env::args()
|
||||
.skip(1)
|
||||
.filter_map(|arg| {
|
||||
// Accept "host:port" or just "host" (use default port)
|
||||
if arg.contains(':') {
|
||||
arg.parse().ok()
|
||||
} else {
|
||||
format!("{}:{}", arg, DEFAULT_PEER_PORT).parse().ok()
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
if sensor_addrs.is_empty() {
|
||||
tracing::info!("usage: BLACKWALL_PSK=<key> blackwall-controller <addr:port> [...]");
|
||||
tracing::info!("example: BLACKWALL_PSK=mysecret blackwall-controller 192.168.1.10:9471");
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let mut controller = Controller::new(psk.as_bytes());
|
||||
tracing::info!(node_id = %controller.node_id, sensors = sensor_addrs.len(), "connecting to sensors");
|
||||
|
||||
// Initial connection to all sensors
|
||||
for addr in &sensor_addrs {
|
||||
if let Err(e) = controller.connect_sensor(*addr).await {
|
||||
tracing::warn!(%addr, "failed to connect to sensor: {}", e);
|
||||
}
|
||||
}
|
||||
|
||||
controller.print_status();
|
||||
|
||||
// Main loop: periodic status reports + reconnection
|
||||
let mut report_interval = tokio::time::interval(REPORT_INTERVAL);
|
||||
let mut heartbeat_interval = tokio::time::interval(HEARTBEAT_INTERVAL);
|
||||
|
||||
loop {
|
||||
tokio::select! {
|
||||
_ = report_interval.tick() => {
|
||||
controller.print_status();
|
||||
}
|
||||
_ = heartbeat_interval.tick() => {
|
||||
// Send heartbeats to connected sensors and read responses
|
||||
controller.send_heartbeats().await;
|
||||
// Reconnect disconnected sensors
|
||||
for addr in &sensor_addrs {
|
||||
let is_disconnected = controller.sensors
|
||||
.get(addr)
|
||||
.map(|s| !s.connected)
|
||||
.unwrap_or(true);
|
||||
if is_disconnected {
|
||||
if let Err(e) = controller.connect_sensor(*addr).await {
|
||||
tracing::debug!(%addr, "reconnect failed: {}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
_ = tokio::signal::ctrl_c() => {
|
||||
tracing::info!("shutting down");
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
257
blackwall-ebpf/Cargo.lock
generated
Executable file
257
blackwall-ebpf/Cargo.lock
generated
Executable file
|
|
@ -0,0 +1,257 @@
|
|||
# This file is automatically @generated by Cargo.
|
||||
# It is not intended for manual editing.
|
||||
version = 4
|
||||
|
||||
[[package]]
|
||||
name = "anyhow"
|
||||
version = "1.0.102"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c"
|
||||
|
||||
[[package]]
|
||||
name = "aya-build"
|
||||
version = "0.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "59bc42f3c5ddacc34eca28a420b47e3cbb3f0f484137cb2bf1ad2153d0eae52a"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"cargo_metadata",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aya-ebpf"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d8dbaf5409a1a0982e5c9bdc0f499a55fe5ead39fe9c846012053faf0d404f73"
|
||||
dependencies = [
|
||||
"aya-ebpf-bindings",
|
||||
"aya-ebpf-cty",
|
||||
"aya-ebpf-macros",
|
||||
"rustversion",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aya-ebpf-bindings"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "71ee8e6a617f040d8da7565ec4010aea75e33cda4662f64c019c66ee97d17889"
|
||||
dependencies = [
|
||||
"aya-build",
|
||||
"aya-ebpf-cty",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aya-ebpf-cty"
|
||||
version = "0.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e6f33396742e7fd0f519c1e0de5141d84e1a8df69146a557c08cc222b0ceace4"
|
||||
dependencies = [
|
||||
"aya-build",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aya-ebpf-macros"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "96fd02363736177e7e91d6c95d7effbca07be87502c7b5b32fc194aed8b177a0"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"proc-macro2-diagnostics",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "blackwall-ebpf"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"aya-ebpf",
|
||||
"common",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "camino"
|
||||
version = "1.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e629a66d692cb9ff1a1c664e41771b3dcaf961985a9774c0eb0bd1b51cf60a48"
|
||||
dependencies = [
|
||||
"serde_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cargo-platform"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "87a0c0e6148f11f01f32650a2ea02d532b2ad4e81d8bd41e6e565b5adc5e6082"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"serde_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cargo_metadata"
|
||||
version = "0.23.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ef987d17b0a113becdd19d3d0022d04d7ef41f9efe4f3fb63ac44ba61df3ade9"
|
||||
dependencies = [
|
||||
"camino",
|
||||
"cargo-platform",
|
||||
"semver",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"thiserror",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "common"
|
||||
version = "0.1.0"
|
||||
|
||||
[[package]]
|
||||
name = "itoa"
|
||||
version = "1.0.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.106"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"
|
||||
dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2-diagnostics"
|
||||
version = "0.10.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"version_check",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.45"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustversion"
|
||||
version = "1.0.22"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d"
|
||||
|
||||
[[package]]
|
||||
name = "semver"
|
||||
version = "1.0.27"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"serde_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.228"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
|
||||
dependencies = [
|
||||
"serde_core",
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_core"
|
||||
version = "1.0.228"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
|
||||
dependencies = [
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive"
|
||||
version = "1.0.228"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_json"
|
||||
version = "1.0.149"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86"
|
||||
dependencies = [
|
||||
"itoa",
|
||||
"memchr",
|
||||
"serde",
|
||||
"serde_core",
|
||||
"zmij",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "2.0.117"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thiserror"
|
||||
version = "2.0.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4"
|
||||
dependencies = [
|
||||
"thiserror-impl",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thiserror-impl"
|
||||
version = "2.0.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "unicode-ident"
|
||||
version = "1.0.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
|
||||
|
||||
[[package]]
|
||||
name = "version_check"
|
||||
version = "0.9.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
|
||||
|
||||
[[package]]
|
||||
name = "zmij"
|
||||
version = "1.0.21"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa"
|
||||
21
blackwall-ebpf/Cargo.toml
Executable file
21
blackwall-ebpf/Cargo.toml
Executable file
|
|
@ -0,0 +1,21 @@
|
|||
[package]
|
||||
name = "blackwall-ebpf"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
|
||||
[[bin]]
|
||||
name = "blackwall-ebpf"
|
||||
path = "src/main.rs"
|
||||
|
||||
|
||||
[dependencies]
|
||||
common = { path = "../common", default-features = false }
|
||||
aya-ebpf = "0.1"
|
||||
|
||||
[profile.release]
|
||||
lto = true
|
||||
panic = "abort"
|
||||
codegen-units = 1
|
||||
opt-level = 2
|
||||
strip = "none"
|
||||
debug = 2
|
||||
3
blackwall-ebpf/rust-toolchain.toml
Executable file
3
blackwall-ebpf/rust-toolchain.toml
Executable file
|
|
@ -0,0 +1,3 @@
|
|||
[toolchain]
|
||||
channel = "nightly"
|
||||
components = ["rust-src"]
|
||||
1334
blackwall-ebpf/src/main.rs
Executable file
1334
blackwall-ebpf/src/main.rs
Executable file
File diff suppressed because it is too large
Load diff
33
blackwall/Cargo.toml
Executable file
33
blackwall/Cargo.toml
Executable file
|
|
@ -0,0 +1,33 @@
|
|||
[package]
|
||||
name = "blackwall"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
|
||||
[features]
|
||||
default = []
|
||||
# Legacy iptables DNAT fallback. V2.0 uses eBPF native DNAT exclusively.
|
||||
iptables-legacy = []
|
||||
|
||||
[[bin]]
|
||||
name = "blackwall"
|
||||
path = "src/main.rs"
|
||||
|
||||
[dependencies]
|
||||
common = { workspace = true }
|
||||
aya = { workspace = true }
|
||||
tokio = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
tracing-subscriber = { workspace = true }
|
||||
anyhow = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
toml = { workspace = true }
|
||||
papaya = { workspace = true }
|
||||
crossbeam-queue = { workspace = true }
|
||||
nix = { workspace = true }
|
||||
rand = { workspace = true }
|
||||
hyper = { workspace = true }
|
||||
hyper-util = { workspace = true }
|
||||
hyper-rustls = { workspace = true }
|
||||
http-body-util = { workspace = true }
|
||||
ring = { workspace = true }
|
||||
74
blackwall/src/ai/batch.rs
Executable file
74
blackwall/src/ai/batch.rs
Executable file
|
|
@ -0,0 +1,74 @@
|
|||
use common::PacketEvent;
|
||||
use std::collections::HashMap;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
/// Batches PacketEvents by source IP, with time-window flushing.
|
||||
pub struct EventBatcher {
|
||||
/// Events grouped by src_ip, with timestamp of first event.
|
||||
pending: HashMap<u32, BatchEntry>,
|
||||
/// Max events per batch before forced flush.
|
||||
max_batch_size: usize,
|
||||
/// Time window before auto-flush.
|
||||
window_duration: Duration,
|
||||
}
|
||||
|
||||
struct BatchEntry {
|
||||
events: Vec<PacketEvent>,
|
||||
first_seen: Instant,
|
||||
}
|
||||
|
||||
impl EventBatcher {
|
||||
/// Create a new batcher with given limits.
|
||||
pub fn new(max_batch_size: usize, window_secs: u64) -> Self {
|
||||
Self {
|
||||
pending: HashMap::new(),
|
||||
max_batch_size,
|
||||
window_duration: Duration::from_secs(window_secs),
|
||||
}
|
||||
}
|
||||
|
||||
/// Add an event to the batch. Returns `Some(batch)` if the batch is ready
|
||||
/// for classification (hit max size).
|
||||
pub fn push(&mut self, event: PacketEvent) -> Option<Vec<PacketEvent>> {
|
||||
let ip = event.src_ip;
|
||||
let entry = self.pending.entry(ip).or_insert_with(|| BatchEntry {
|
||||
events: Vec::new(),
|
||||
first_seen: Instant::now(),
|
||||
});
|
||||
|
||||
entry.events.push(event);
|
||||
|
||||
if entry.events.len() >= self.max_batch_size {
|
||||
let batch = self.pending.remove(&ip).map(|e| e.events);
|
||||
return batch;
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
/// Discard any pending batch for a given IP (e.g., after blocking it).
|
||||
pub fn discard_ip(&mut self, ip: u32) {
|
||||
self.pending.remove(&ip);
|
||||
}
|
||||
|
||||
/// Flush all batches older than the window duration.
|
||||
pub fn flush_expired(&mut self) -> Vec<(u32, Vec<PacketEvent>)> {
|
||||
let now = Instant::now();
|
||||
let mut flushed = Vec::new();
|
||||
let mut expired_keys = Vec::new();
|
||||
|
||||
for (&ip, entry) in &self.pending {
|
||||
if now.duration_since(entry.first_seen) >= self.window_duration {
|
||||
expired_keys.push(ip);
|
||||
}
|
||||
}
|
||||
|
||||
for ip in expired_keys {
|
||||
if let Some(entry) = self.pending.remove(&ip) {
|
||||
flushed.push((ip, entry.events));
|
||||
}
|
||||
}
|
||||
|
||||
flushed
|
||||
}
|
||||
}
|
||||
365
blackwall/src/ai/classifier.rs
Executable file
365
blackwall/src/ai/classifier.rs
Executable file
|
|
@ -0,0 +1,365 @@
|
|||
use common::PacketEvent;
|
||||
use std::collections::HashSet;
|
||||
|
||||
use crate::ai::client::OllamaClient;
|
||||
|
||||
/// System prompt for threat classification. Low temperature, structured output.
|
||||
pub const CLASSIFICATION_SYSTEM_PROMPT: &str = r#"You are a network security analyst.
|
||||
Analyze the following traffic summary and classify the threat.
|
||||
Respond with EXACTLY one line in this format:
|
||||
VERDICT:<Benign|Suspicious|Malicious> CATEGORY:<category> CONFIDENCE:<0.0-1.0>
|
||||
|
||||
Categories: DDoS_SYN_Flood, DDoS_UDP_Flood, Port_Scan, Brute_Force,
|
||||
Exploit, C2_Communication, Data_Exfiltration, Other
|
||||
|
||||
Example: VERDICT:Malicious CATEGORY:Port_Scan CONFIDENCE:0.85"#;
|
||||
|
||||
/// Classification verdict from the AI module.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum ThreatVerdict {
|
||||
/// Normal traffic.
|
||||
Benign,
|
||||
/// Needs monitoring but not yet actionable.
|
||||
Suspicious { reason: String, confidence: f32 },
|
||||
/// Confirmed threat — take action.
|
||||
Malicious {
|
||||
category: ThreatCategory,
|
||||
confidence: f32,
|
||||
},
|
||||
/// LLM unavailable, deterministic rules didn't match.
|
||||
Unknown,
|
||||
}
|
||||
|
||||
/// Threat categories for classification.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum ThreatCategory {
|
||||
DdosSynFlood,
|
||||
DdosUdpFlood,
|
||||
PortScan,
|
||||
BruteForce,
|
||||
Exploit,
|
||||
C2Communication,
|
||||
DataExfiltration,
|
||||
Other(String),
|
||||
}
|
||||
|
||||
/// Classifies batched events using deterministic rules + LLM fallback.
|
||||
pub struct ThreatClassifier {
|
||||
client: OllamaClient,
|
||||
}
|
||||
|
||||
impl ThreatClassifier {
|
||||
/// Create a new classifier backed by the given Ollama client.
|
||||
pub fn new(client: OllamaClient) -> Self {
|
||||
Self { client }
|
||||
}
|
||||
|
||||
/// Get a reference to the underlying Ollama client.
|
||||
pub fn client(&self) -> &OllamaClient {
|
||||
&self.client
|
||||
}
|
||||
|
||||
/// Classify a batch of events from the same source IP.
|
||||
pub async fn classify(&self, events: &[PacketEvent]) -> ThreatVerdict {
|
||||
if events.is_empty() {
|
||||
return ThreatVerdict::Benign;
|
||||
}
|
||||
|
||||
// 1. Quick deterministic check
|
||||
if let Some(verdict) = self.deterministic_classify(events) {
|
||||
return verdict;
|
||||
}
|
||||
|
||||
// 2. If LLM unavailable, return Unknown
|
||||
if !self.client.is_available() {
|
||||
return ThreatVerdict::Unknown;
|
||||
}
|
||||
|
||||
// 3. Build prompt and query LLM
|
||||
let prompt = self.build_classification_prompt(events);
|
||||
match self.client.classify_threat(&prompt).await {
|
||||
Ok(response) => self.parse_llm_response(&response),
|
||||
Err(_) => ThreatVerdict::Unknown,
|
||||
}
|
||||
}
|
||||
|
||||
/// Deterministic fallback classification (no LLM needed).
|
||||
fn deterministic_classify(&self, events: &[PacketEvent]) -> Option<ThreatVerdict> {
|
||||
let count = events.len() as u32;
|
||||
if count == 0 {
|
||||
return None;
|
||||
}
|
||||
|
||||
let avg_entropy = events.iter().map(|e| e.entropy_score).sum::<u32>() / count;
|
||||
|
||||
// Very high entropy + many events → encrypted attack payload
|
||||
if avg_entropy > 7500 && events.len() > 50 {
|
||||
return Some(ThreatVerdict::Malicious {
|
||||
category: ThreatCategory::Exploit,
|
||||
confidence: 0.7,
|
||||
});
|
||||
}
|
||||
|
||||
// SYN flood detection: many SYN without ACK
|
||||
let syn_count = events
|
||||
.iter()
|
||||
.filter(|e| e.flags & 0x02 != 0 && e.flags & 0x10 == 0)
|
||||
.count();
|
||||
if syn_count > 100 {
|
||||
return Some(ThreatVerdict::Malicious {
|
||||
category: ThreatCategory::DdosSynFlood,
|
||||
confidence: 0.9,
|
||||
});
|
||||
}
|
||||
|
||||
// Port scan detection: many unique destination ports
|
||||
let unique_ports: HashSet<u16> = events.iter().map(|e| e.dst_port).collect();
|
||||
if unique_ports.len() > 20 {
|
||||
return Some(ThreatVerdict::Malicious {
|
||||
category: ThreatCategory::PortScan,
|
||||
confidence: 0.85,
|
||||
});
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
fn build_classification_prompt(&self, events: &[PacketEvent]) -> String {
|
||||
let src_ip = common::util::ip_from_u32(events[0].src_ip);
|
||||
let event_count = events.len();
|
||||
let avg_entropy = events.iter().map(|e| e.entropy_score).sum::<u32>() / event_count as u32;
|
||||
|
||||
let unique_dst_ports: HashSet<u16> = events.iter().map(|e| e.dst_port).collect();
|
||||
|
||||
let syn_count = events.iter().filter(|e| e.flags & 0x02 != 0).count();
|
||||
let ack_count = events.iter().filter(|e| e.flags & 0x10 != 0).count();
|
||||
let rst_count = events.iter().filter(|e| e.flags & 0x04 != 0).count();
|
||||
|
||||
let tcp_count = events.iter().filter(|e| e.protocol == 6).count();
|
||||
let udp_count = events.iter().filter(|e| e.protocol == 17).count();
|
||||
|
||||
format!(
|
||||
"Source IP: {}\nEvent count: {} in last 10s\n\
|
||||
Avg byte diversity: {:.1}/8.0\nProtocols: TCP={}, UDP={}\n\
|
||||
Unique dst ports: {}\n\
|
||||
TCP flags: SYN={}, ACK={}, RST={}",
|
||||
src_ip,
|
||||
event_count,
|
||||
avg_entropy as f64 / 1000.0,
|
||||
tcp_count,
|
||||
udp_count,
|
||||
unique_dst_ports.len(),
|
||||
syn_count,
|
||||
ack_count,
|
||||
rst_count,
|
||||
)
|
||||
}
|
||||
|
||||
fn parse_llm_response(&self, response: &str) -> ThreatVerdict {
|
||||
// Parse format: "VERDICT:Malicious CATEGORY:Port_Scan CONFIDENCE:0.85"
|
||||
let line = response.lines().find(|l| l.starts_with("VERDICT:"));
|
||||
let line = match line {
|
||||
Some(l) => l,
|
||||
None => return ThreatVerdict::Unknown,
|
||||
};
|
||||
|
||||
let mut verdict_str = "";
|
||||
let mut category_str = "";
|
||||
let mut confidence: f32 = 0.0;
|
||||
|
||||
for part in line.split_whitespace() {
|
||||
if let Some(v) = part.strip_prefix("VERDICT:") {
|
||||
verdict_str = v;
|
||||
} else if let Some(c) = part.strip_prefix("CATEGORY:") {
|
||||
category_str = c;
|
||||
} else if let Some(conf) = part.strip_prefix("CONFIDENCE:") {
|
||||
confidence = conf.parse().unwrap_or(0.0);
|
||||
}
|
||||
}
|
||||
|
||||
match verdict_str {
|
||||
"Benign" => ThreatVerdict::Benign,
|
||||
"Suspicious" => ThreatVerdict::Suspicious {
|
||||
reason: category_str.to_string(),
|
||||
confidence,
|
||||
},
|
||||
"Malicious" => {
|
||||
let category = match category_str {
|
||||
"DDoS_SYN_Flood" => ThreatCategory::DdosSynFlood,
|
||||
"DDoS_UDP_Flood" => ThreatCategory::DdosUdpFlood,
|
||||
"Port_Scan" => ThreatCategory::PortScan,
|
||||
"Brute_Force" => ThreatCategory::BruteForce,
|
||||
"Exploit" => ThreatCategory::Exploit,
|
||||
"C2_Communication" => ThreatCategory::C2Communication,
|
||||
"Data_Exfiltration" => ThreatCategory::DataExfiltration,
|
||||
other => ThreatCategory::Other(other.to_string()),
|
||||
};
|
||||
ThreatVerdict::Malicious {
|
||||
category,
|
||||
confidence,
|
||||
}
|
||||
}
|
||||
_ => ThreatVerdict::Unknown,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::ai::client::OllamaClient;
|
||||
|
||||
fn test_classifier() -> ThreatClassifier {
|
||||
let client = OllamaClient::new(
|
||||
"http://localhost:11434".into(),
|
||||
"test".into(),
|
||||
"test".into(),
|
||||
1000,
|
||||
);
|
||||
ThreatClassifier::new(client)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_malicious_verdict() {
|
||||
let c = test_classifier();
|
||||
let resp = "VERDICT:Malicious CATEGORY:Port_Scan CONFIDENCE:0.85";
|
||||
match c.parse_llm_response(resp) {
|
||||
ThreatVerdict::Malicious {
|
||||
category,
|
||||
confidence,
|
||||
} => {
|
||||
assert_eq!(category, ThreatCategory::PortScan);
|
||||
assert!((confidence - 0.85).abs() < 0.01);
|
||||
}
|
||||
other => panic!("expected Malicious, got {:?}", other),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_benign_verdict() {
|
||||
let c = test_classifier();
|
||||
let resp = "VERDICT:Benign CATEGORY:None CONFIDENCE:0.95";
|
||||
assert_eq!(c.parse_llm_response(resp), ThreatVerdict::Benign);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_suspicious_verdict() {
|
||||
let c = test_classifier();
|
||||
let resp = "VERDICT:Suspicious CATEGORY:Brute_Force CONFIDENCE:0.6";
|
||||
match c.parse_llm_response(resp) {
|
||||
ThreatVerdict::Suspicious { reason, confidence } => {
|
||||
assert_eq!(reason, "Brute_Force");
|
||||
assert!((confidence - 0.6).abs() < 0.01);
|
||||
}
|
||||
other => panic!("expected Suspicious, got {:?}", other),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_unknown_on_garbage() {
|
||||
let c = test_classifier();
|
||||
assert_eq!(
|
||||
c.parse_llm_response("some random LLM output"),
|
||||
ThreatVerdict::Unknown
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_unknown_on_empty() {
|
||||
let c = test_classifier();
|
||||
assert_eq!(c.parse_llm_response(""), ThreatVerdict::Unknown);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_multiline_finds_verdict() {
|
||||
let c = test_classifier();
|
||||
let resp =
|
||||
"Analyzing traffic...\nVERDICT:Malicious CATEGORY:DDoS_SYN_Flood CONFIDENCE:0.9\nDone.";
|
||||
match c.parse_llm_response(resp) {
|
||||
ThreatVerdict::Malicious { category, .. } => {
|
||||
assert_eq!(category, ThreatCategory::DdosSynFlood);
|
||||
}
|
||||
other => panic!("expected Malicious, got {:?}", other),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deterministic_syn_flood_detection() {
|
||||
let c = test_classifier();
|
||||
// Generate 120 SYN-only events (flags = 0x02)
|
||||
let events: Vec<PacketEvent> = (0..120)
|
||||
.map(|_| PacketEvent {
|
||||
src_ip: 0x0A000001,
|
||||
dst_ip: 0x0A000002,
|
||||
src_port: 12345,
|
||||
dst_port: 80,
|
||||
protocol: 6,
|
||||
flags: 0x02, // SYN only
|
||||
payload_len: 0,
|
||||
entropy_score: 1000,
|
||||
timestamp_ns: 0,
|
||||
_padding: 0,
|
||||
packet_size: 64,
|
||||
})
|
||||
.collect();
|
||||
|
||||
match c.deterministic_classify(&events) {
|
||||
Some(ThreatVerdict::Malicious { category, .. }) => {
|
||||
assert_eq!(category, ThreatCategory::DdosSynFlood);
|
||||
}
|
||||
other => panic!("expected SYN flood, got {:?}", other),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deterministic_port_scan_detection() {
|
||||
let c = test_classifier();
|
||||
// Generate events to 25 unique destination ports
|
||||
let events: Vec<PacketEvent> = (0..25)
|
||||
.map(|i| PacketEvent {
|
||||
src_ip: 0x0A000001,
|
||||
dst_ip: 0x0A000002,
|
||||
src_port: 12345,
|
||||
dst_port: 1000 + i as u16,
|
||||
protocol: 6,
|
||||
flags: 0x02,
|
||||
payload_len: 64,
|
||||
entropy_score: 3000,
|
||||
timestamp_ns: 0,
|
||||
_padding: 0,
|
||||
packet_size: 128,
|
||||
})
|
||||
.collect();
|
||||
|
||||
match c.deterministic_classify(&events) {
|
||||
Some(ThreatVerdict::Malicious { category, .. }) => {
|
||||
assert_eq!(category, ThreatCategory::PortScan);
|
||||
}
|
||||
other => panic!("expected PortScan, got {:?}", other),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deterministic_benign_traffic() {
|
||||
let c = test_classifier();
|
||||
// Few events, normal entropy, single port
|
||||
let events: Vec<PacketEvent> = (0..5)
|
||||
.map(|_| PacketEvent {
|
||||
src_ip: 0x0A000001,
|
||||
dst_ip: 0x0A000002,
|
||||
src_port: 12345,
|
||||
dst_port: 443,
|
||||
protocol: 6,
|
||||
flags: 0x12, // SYN+ACK
|
||||
payload_len: 64,
|
||||
entropy_score: 3000,
|
||||
timestamp_ns: 0,
|
||||
_padding: 0,
|
||||
packet_size: 128,
|
||||
})
|
||||
.collect();
|
||||
|
||||
// Should return None (no deterministic match → falls through to LLM)
|
||||
assert!(c.deterministic_classify(&events).is_none());
|
||||
}
|
||||
}
|
||||
151
blackwall/src/ai/client.rs
Executable file
151
blackwall/src/ai/client.rs
Executable file
|
|
@ -0,0 +1,151 @@
|
|||
use anyhow::{Context, Result};
|
||||
use http_body_util::{BodyExt, Full};
|
||||
use hyper::body::Bytes;
|
||||
use hyper::Request;
|
||||
use hyper_util::client::legacy::Client;
|
||||
use hyper_util::rt::TokioExecutor;
|
||||
use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
|
||||
use std::time::Duration;
|
||||
|
||||
/// Max concurrent LLM requests. Prevents queue buildup under DDoS when many
|
||||
/// unique IPs generate batches faster than Ollama can classify them.
|
||||
/// Excess requests are shed (return Unknown verdict) rather than queued.
|
||||
const MAX_CONCURRENT_LLM_REQUESTS: u32 = 2;
|
||||
|
||||
/// HTTP client for the Ollama REST API with backpressure.
|
||||
pub struct OllamaClient {
|
||||
base_url: String,
|
||||
model: String,
|
||||
fallback_model: String,
|
||||
timeout: Duration,
|
||||
available: AtomicBool,
|
||||
/// Tracks in-flight LLM requests for backpressure.
|
||||
in_flight: AtomicU32,
|
||||
/// Counter of requests shed due to backpressure.
|
||||
shed_count: AtomicU32,
|
||||
}
|
||||
|
||||
impl OllamaClient {
|
||||
/// Create a new client from AI config values.
|
||||
pub fn new(base_url: String, model: String, fallback_model: String, timeout_ms: u64) -> Self {
|
||||
Self {
|
||||
base_url,
|
||||
model,
|
||||
fallback_model,
|
||||
timeout: Duration::from_millis(timeout_ms),
|
||||
available: AtomicBool::new(false),
|
||||
in_flight: AtomicU32::new(0),
|
||||
shed_count: AtomicU32::new(0),
|
||||
}
|
||||
}
|
||||
|
||||
/// Number of requests shed due to backpressure since start.
|
||||
#[allow(dead_code)]
|
||||
pub fn shed_count(&self) -> u32 {
|
||||
self.shed_count.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Check if Ollama is reachable (GET /api/tags).
|
||||
pub async fn health_check(&self) -> bool {
|
||||
let client = Client::builder(TokioExecutor::new()).build_http();
|
||||
let url = format!("{}/api/tags", self.base_url);
|
||||
let req = match Request::get(&url).body(http_body_util::Empty::<Bytes>::new()) {
|
||||
Ok(r) => r,
|
||||
Err(_) => return false,
|
||||
};
|
||||
|
||||
let result = tokio::time::timeout(Duration::from_secs(3), client.request(req)).await;
|
||||
let ok = matches!(result, Ok(Ok(resp)) if resp.status().is_success());
|
||||
self.available.store(ok, Ordering::Relaxed);
|
||||
ok
|
||||
}
|
||||
|
||||
/// Whether the last health check succeeded.
|
||||
pub fn is_available(&self) -> bool {
|
||||
self.available.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Send a classification prompt to the LLM. Tries primary, then fallback.
|
||||
///
|
||||
/// Applies backpressure: if `MAX_CONCURRENT_LLM_REQUESTS` are already
|
||||
/// in-flight, returns an error immediately (load shedding).
|
||||
pub async fn classify_threat(&self, prompt: &str) -> Result<String> {
|
||||
// Backpressure: reject if too many in-flight requests
|
||||
let current = self.in_flight.fetch_add(1, Ordering::Relaxed);
|
||||
if current >= MAX_CONCURRENT_LLM_REQUESTS {
|
||||
self.in_flight.fetch_sub(1, Ordering::Relaxed);
|
||||
let shed = self.shed_count.fetch_add(1, Ordering::Relaxed) + 1;
|
||||
tracing::debug!(in_flight = current, total_shed = shed,
|
||||
"LLM backpressure — request shed");
|
||||
anyhow::bail!("LLM backpressure: {} in-flight, request shed", current);
|
||||
}
|
||||
|
||||
let result = self.classify_inner(prompt).await;
|
||||
self.in_flight.fetch_sub(1, Ordering::Relaxed);
|
||||
result
|
||||
}
|
||||
|
||||
/// Inner classification logic (primary + fallback).
|
||||
async fn classify_inner(&self, prompt: &str) -> Result<String> {
|
||||
let body = self.build_body(prompt, &self.model)?;
|
||||
match self.send(&body).await {
|
||||
Ok(r) => Ok(r),
|
||||
Err(e) => {
|
||||
tracing::warn!("primary model failed: {}, trying fallback", e);
|
||||
let fallback_body = self.build_body(prompt, &self.fallback_model)?;
|
||||
self.send(&fallback_body).await
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn build_body(&self, prompt: &str, model: &str) -> Result<Vec<u8>> {
|
||||
let body = serde_json::json!({
|
||||
"model": model,
|
||||
"messages": [
|
||||
{"role": "system", "content": super::classifier::CLASSIFICATION_SYSTEM_PROMPT},
|
||||
{"role": "user", "content": prompt},
|
||||
],
|
||||
"stream": false,
|
||||
"options": {
|
||||
"num_predict": 256,
|
||||
"temperature": 0.1,
|
||||
},
|
||||
});
|
||||
serde_json::to_vec(&body).context("serialize request")
|
||||
}
|
||||
|
||||
async fn send(&self, body: &[u8]) -> Result<String> {
|
||||
let client = Client::builder(TokioExecutor::new()).build_http();
|
||||
let req = Request::post(format!("{}/api/chat", self.base_url))
|
||||
.header("Content-Type", "application/json")
|
||||
.body(Full::new(Bytes::from(body.to_vec())))
|
||||
.context("build request")?;
|
||||
|
||||
// Wrap BOTH the request send AND body read in a single timeout.
|
||||
// Without this, a slowloris-style response from Ollama (infinitely
|
||||
// slow body) hangs forever, in_flight never decrements, and after
|
||||
// MAX_CONCURRENT_LLM_REQUESTS such requests the AI pipeline is dead.
|
||||
let bytes = tokio::time::timeout(self.timeout, async {
|
||||
let resp = client
|
||||
.request(req)
|
||||
.await
|
||||
.context("HTTP request failed")?;
|
||||
let collected = resp
|
||||
.into_body()
|
||||
.collect()
|
||||
.await
|
||||
.context("read response body")?
|
||||
.to_bytes();
|
||||
Ok::<_, anyhow::Error>(collected)
|
||||
})
|
||||
.await
|
||||
.context("LLM request+response timed out")??;
|
||||
|
||||
let json: serde_json::Value = serde_json::from_slice(&bytes).context("invalid JSON")?;
|
||||
|
||||
json["message"]["content"]
|
||||
.as_str()
|
||||
.map(|s| s.to_string())
|
||||
.context("missing content in response")
|
||||
}
|
||||
}
|
||||
3
blackwall/src/ai/mod.rs
Executable file
3
blackwall/src/ai/mod.rs
Executable file
|
|
@ -0,0 +1,3 @@
|
|||
pub mod batch;
|
||||
pub mod classifier;
|
||||
pub mod client;
|
||||
159
blackwall/src/antifingerprint.rs
Executable file
159
blackwall/src/antifingerprint.rs
Executable file
|
|
@ -0,0 +1,159 @@
|
|||
//! Anti-fingerprinting: evade attacker reconnaissance.
|
||||
//!
|
||||
//! Randomizes observable characteristics to prevent attackers from
|
||||
//! identifying Blackwall's presence through response timing, error
|
||||
//! messages, or behavior patterns.
|
||||
|
||||
use rand::rngs::StdRng;
|
||||
use rand::{Rng, SeedableRng};
|
||||
use std::time::Duration;
|
||||
|
||||
/// Jitter range for response timing (ms).
|
||||
const MIN_JITTER_MS: u64 = 10;
|
||||
const MAX_JITTER_MS: u64 = 500;
|
||||
|
||||
/// Pool of fake server banners for HTTP responses.
|
||||
const HTTP_SERVER_BANNERS: &[&str] = &[
|
||||
"Apache/2.4.58 (Ubuntu)",
|
||||
"Apache/2.4.57 (Debian)",
|
||||
"nginx/1.24.0",
|
||||
"nginx/1.26.0 (Ubuntu)",
|
||||
"Microsoft-IIS/10.0",
|
||||
"LiteSpeed",
|
||||
"openresty/1.25.3.1",
|
||||
"Caddy",
|
||||
];
|
||||
|
||||
/// Pool of fake SSH banners.
|
||||
const SSH_BANNERS: &[&str] = &[
|
||||
"SSH-2.0-OpenSSH_9.6p1 Ubuntu-3ubuntu13.5",
|
||||
"SSH-2.0-OpenSSH_9.7p1 Debian-5",
|
||||
"SSH-2.0-OpenSSH_8.9p1 Ubuntu-3ubuntu0.10",
|
||||
"SSH-2.0-OpenSSH_9.3p1 Ubuntu-1ubuntu3.6",
|
||||
"SSH-2.0-dropbear_2022.82",
|
||||
];
|
||||
|
||||
/// Pool of fake MySQL version strings.
|
||||
const MYSQL_VERSIONS: &[&str] = &[
|
||||
"8.0.36-0ubuntu0.24.04.1",
|
||||
"8.0.35-0ubuntu0.22.04.1",
|
||||
"8.0.37",
|
||||
"5.7.44-log",
|
||||
"10.11.6-MariaDB",
|
||||
];
|
||||
|
||||
/// Pool of fake operating system identifiers (for SSH comments).
|
||||
const OS_COMMENTS: &[&str] = &[
|
||||
"Ubuntu-3ubuntu13.5",
|
||||
"Debian-5+deb12u1",
|
||||
"Ubuntu-1ubuntu3.6",
|
||||
"FreeBSD-20240806",
|
||||
];
|
||||
|
||||
/// Anti-fingerprinting profile: randomized per-session.
|
||||
pub struct AntiFingerprintProfile {
|
||||
rng: StdRng,
|
||||
/// Selected HTTP server banner for this session
|
||||
pub http_banner: &'static str,
|
||||
/// Selected SSH banner for this session
|
||||
pub ssh_banner: &'static str,
|
||||
/// Selected MySQL version for this session
|
||||
pub mysql_version: &'static str,
|
||||
/// Selected OS comment for this session
|
||||
pub os_comment: &'static str,
|
||||
}
|
||||
|
||||
impl AntiFingerprintProfile {
|
||||
/// Create a new randomized profile.
|
||||
pub fn new() -> Self {
|
||||
let mut rng = StdRng::from_entropy();
|
||||
let http_banner = HTTP_SERVER_BANNERS[rng.gen_range(0..HTTP_SERVER_BANNERS.len())];
|
||||
let ssh_banner = SSH_BANNERS[rng.gen_range(0..SSH_BANNERS.len())];
|
||||
let mysql_version = MYSQL_VERSIONS[rng.gen_range(0..MYSQL_VERSIONS.len())];
|
||||
let os_comment = OS_COMMENTS[rng.gen_range(0..OS_COMMENTS.len())];
|
||||
|
||||
Self {
|
||||
rng,
|
||||
http_banner,
|
||||
ssh_banner,
|
||||
mysql_version,
|
||||
os_comment,
|
||||
}
|
||||
}
|
||||
|
||||
/// Generate a random delay to add to a response (anti-timing-analysis).
|
||||
pub fn response_jitter(&mut self) -> Duration {
|
||||
Duration::from_millis(self.rng.gen_range(MIN_JITTER_MS..=MAX_JITTER_MS))
|
||||
}
|
||||
|
||||
/// Randomly decide whether to add a fake header to an HTTP response.
|
||||
pub fn should_add_fake_header(&mut self) -> bool {
|
||||
self.rng.gen_ratio(1, 3) // 33% chance
|
||||
}
|
||||
|
||||
/// Generate a random fake HTTP header.
|
||||
pub fn fake_http_header(&mut self) -> (&'static str, String) {
|
||||
let headers = [
|
||||
("X-Powered-By", vec!["PHP/8.3.6", "PHP/8.2.18", "ASP.NET", "Express"]),
|
||||
("X-Cache", vec!["HIT", "MISS", "HIT from cdn-edge-01"]),
|
||||
("Via", vec!["1.1 varnish", "1.1 squid", "HTTP/1.1 cloudfront"]),
|
||||
];
|
||||
let (name, values) = &headers[self.rng.gen_range(0..headers.len())];
|
||||
let value = values[self.rng.gen_range(0..values.len())].to_string();
|
||||
(name, value)
|
||||
}
|
||||
|
||||
/// Randomly corrupt a timestamp to prevent timing attacks.
|
||||
pub fn fuzz_timestamp(&mut self, base_secs: u64) -> u64 {
|
||||
let drift = self.rng.gen_range(0..=5);
|
||||
base_secs.wrapping_add(drift)
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for AntiFingerprintProfile {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn profile_randomization() {
|
||||
let p1 = AntiFingerprintProfile::new();
|
||||
// Just verify it doesn't panic and produces valid strings
|
||||
assert!(!p1.http_banner.is_empty());
|
||||
assert!(p1.ssh_banner.starts_with("SSH-2.0-"));
|
||||
assert!(!p1.mysql_version.is_empty());
|
||||
assert!(!p1.os_comment.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn jitter_in_range() {
|
||||
let mut profile = AntiFingerprintProfile::new();
|
||||
for _ in 0..100 {
|
||||
let jitter = profile.response_jitter();
|
||||
assert!(jitter.as_millis() >= MIN_JITTER_MS as u128);
|
||||
assert!(jitter.as_millis() <= MAX_JITTER_MS as u128);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fake_header_generation() {
|
||||
let mut profile = AntiFingerprintProfile::new();
|
||||
let (name, value) = profile.fake_http_header();
|
||||
assert!(!name.is_empty());
|
||||
assert!(!value.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn timestamp_fuzzing() {
|
||||
let mut profile = AntiFingerprintProfile::new();
|
||||
let base = 1000u64;
|
||||
let fuzzed = profile.fuzz_timestamp(base);
|
||||
assert!(fuzzed >= base);
|
||||
assert!(fuzzed <= base + 5);
|
||||
}
|
||||
}
|
||||
12
blackwall/src/behavior/mod.rs
Executable file
12
blackwall/src/behavior/mod.rs
Executable file
|
|
@ -0,0 +1,12 @@
|
|||
//! Behavioral engine: per-IP state machine for threat progression tracking.
|
||||
//!
|
||||
//! Each source IP gets a `BehaviorProfile` that tracks packet statistics,
|
||||
//! connection patterns, and a phase in the threat lifecycle. Transitions
|
||||
//! are driven by deterministic thresholds (fast path) with optional
|
||||
//! LLM-assisted classification (slow path).
|
||||
|
||||
mod profile;
|
||||
mod transitions;
|
||||
|
||||
pub use profile::{BehaviorPhase, BehaviorProfile};
|
||||
pub use transitions::{TransitionVerdict, evaluate_transitions};
|
||||
430
blackwall/src/behavior/profile.rs
Executable file
430
blackwall/src/behavior/profile.rs
Executable file
|
|
@ -0,0 +1,430 @@
|
|||
//! Per-IP behavioral profile: statistics and lifecycle phase tracking.
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::time::Instant;
|
||||
|
||||
/// Lifecycle phases for a tracked IP address.
|
||||
/// Transitions are monotonically increasing in suspicion (except demotion to Trusted).
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum BehaviorPhase {
|
||||
/// First contact, insufficient data for classification.
|
||||
New,
|
||||
/// Normal traffic pattern, low suspicion.
|
||||
Normal,
|
||||
/// Elevated trust after sustained benign behavior.
|
||||
Trusted,
|
||||
/// Active reconnaissance (port scanning, service enumeration).
|
||||
Probing,
|
||||
/// Systematic scanning (sequential ports, multiple protocols).
|
||||
Scanning,
|
||||
/// Exploit attempts detected (high entropy payloads, known signatures).
|
||||
Exploiting,
|
||||
/// Established command-and-control pattern (beaconing, exfiltration).
|
||||
EstablishedC2,
|
||||
}
|
||||
|
||||
impl BehaviorPhase {
|
||||
/// Numeric suspicion level for ordering (higher = more suspicious).
|
||||
pub fn suspicion_level(self) -> u8 {
|
||||
match self {
|
||||
Self::Trusted => 0,
|
||||
Self::Normal => 1,
|
||||
Self::New => 2,
|
||||
Self::Probing => 3,
|
||||
Self::Scanning => 4,
|
||||
Self::Exploiting => 5,
|
||||
Self::EstablishedC2 => 6,
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether this phase should trigger active response (block/tarpit).
|
||||
pub fn is_actionable(self) -> bool {
|
||||
matches!(self, Self::Scanning | Self::Exploiting | Self::EstablishedC2)
|
||||
}
|
||||
}
|
||||
|
||||
/// Aggregated behavioral statistics for a single source IP.
|
||||
pub struct BehaviorProfile {
|
||||
/// When this IP was first observed.
|
||||
pub first_seen: Instant,
|
||||
/// When the last packet was observed.
|
||||
pub last_seen: Instant,
|
||||
/// Total packets observed from this IP.
|
||||
pub total_packets: u64,
|
||||
/// Unique destination ports contacted.
|
||||
pub unique_dst_ports: HashSet<u16>,
|
||||
/// TCP SYN packets (connection attempts).
|
||||
pub syn_count: u64,
|
||||
/// TCP ACK packets.
|
||||
pub ack_count: u64,
|
||||
/// TCP RST packets (aborted connections).
|
||||
pub rst_count: u64,
|
||||
/// TCP FIN packets (clean closes).
|
||||
pub fin_count: u64,
|
||||
/// Running sum of entropy scores (for computing average).
|
||||
pub entropy_sum: u64,
|
||||
/// Number of entropy samples collected.
|
||||
pub entropy_samples: u64,
|
||||
/// Current lifecycle phase.
|
||||
pub phase: BehaviorPhase,
|
||||
/// Suspicion score (0.0–1.0), drives escalation thresholds.
|
||||
pub suspicion_score: f32,
|
||||
/// Timestamps of recent packets for inter-arrival analysis (circular, last N).
|
||||
pub recent_timestamps: Vec<u32>,
|
||||
/// Maximum recent timestamps to keep.
|
||||
recent_ts_capacity: usize,
|
||||
/// Index for circular buffer insertion.
|
||||
recent_ts_idx: usize,
|
||||
/// Number of times this IP has been escalated.
|
||||
pub escalation_count: u32,
|
||||
}
|
||||
|
||||
impl BehaviorProfile {
|
||||
/// Create a new profile for a first-seen IP.
|
||||
pub fn new() -> Self {
|
||||
let now = Instant::now();
|
||||
let cap = 64;
|
||||
Self {
|
||||
first_seen: now,
|
||||
last_seen: now,
|
||||
total_packets: 0,
|
||||
unique_dst_ports: HashSet::new(),
|
||||
syn_count: 0,
|
||||
ack_count: 0,
|
||||
rst_count: 0,
|
||||
fin_count: 0,
|
||||
entropy_sum: 0,
|
||||
entropy_samples: 0,
|
||||
phase: BehaviorPhase::New,
|
||||
suspicion_score: 0.0,
|
||||
recent_timestamps: vec![0u32; cap],
|
||||
recent_ts_capacity: cap,
|
||||
recent_ts_idx: 0,
|
||||
escalation_count: 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// Ingest a packet event, updating all counters.
|
||||
pub fn update(&mut self, event: &common::PacketEvent) {
|
||||
self.last_seen = Instant::now();
|
||||
self.total_packets += 1;
|
||||
self.unique_dst_ports.insert(event.dst_port);
|
||||
|
||||
// TCP flag counters (protocol 6 = TCP)
|
||||
if event.protocol == 6 {
|
||||
if event.flags & 0x02 != 0 {
|
||||
self.syn_count += 1;
|
||||
}
|
||||
if event.flags & 0x10 != 0 {
|
||||
self.ack_count += 1;
|
||||
}
|
||||
if event.flags & 0x04 != 0 {
|
||||
self.rst_count += 1;
|
||||
}
|
||||
if event.flags & 0x01 != 0 {
|
||||
self.fin_count += 1;
|
||||
}
|
||||
}
|
||||
|
||||
// Entropy tracking
|
||||
if event.entropy_score > 0 {
|
||||
self.entropy_sum += event.entropy_score as u64;
|
||||
self.entropy_samples += 1;
|
||||
}
|
||||
|
||||
// Circular buffer for inter-arrival times
|
||||
self.recent_timestamps[self.recent_ts_idx] = event.timestamp_ns;
|
||||
self.recent_ts_idx = (self.recent_ts_idx + 1) % self.recent_ts_capacity;
|
||||
}
|
||||
|
||||
/// Average byte diversity score (unique_count × 31 scale), or 0 if no samples.
|
||||
pub fn avg_entropy(&self) -> u32 {
|
||||
if self.entropy_samples == 0 {
|
||||
return 0;
|
||||
}
|
||||
(self.entropy_sum / self.entropy_samples) as u32
|
||||
}
|
||||
|
||||
/// Ratio of SYN-only packets (SYN without ACK) to total packets.
|
||||
pub fn syn_only_ratio(&self) -> f32 {
|
||||
if self.total_packets == 0 {
|
||||
return 0.0;
|
||||
}
|
||||
let syn_only = self.syn_count.saturating_sub(self.ack_count);
|
||||
syn_only as f32 / self.total_packets as f32
|
||||
}
|
||||
|
||||
/// Duration since first observation.
|
||||
pub fn age(&self) -> std::time::Duration {
|
||||
self.last_seen.duration_since(self.first_seen)
|
||||
}
|
||||
|
||||
/// Number of unique destination ports observed.
|
||||
pub fn port_diversity(&self) -> usize {
|
||||
self.unique_dst_ports.len()
|
||||
}
|
||||
|
||||
/// Whether this profile has enough data for meaningful classification.
|
||||
pub fn has_sufficient_data(&self) -> bool {
|
||||
self.total_packets >= 5
|
||||
}
|
||||
|
||||
/// Detect beaconing: regular inter-arrival times (C2 pattern).
|
||||
/// Returns the coefficient of variation (stddev/mean) × 1000.
|
||||
/// Low values (<300) indicate periodic/regular intervals = beaconing.
|
||||
pub fn beaconing_score(&self) -> Option<u32> {
|
||||
if self.total_packets < 20 {
|
||||
return None;
|
||||
}
|
||||
// Collect non-zero timestamps from circular buffer
|
||||
let mut timestamps: Vec<u32> = self.recent_timestamps.iter()
|
||||
.copied()
|
||||
.filter(|&t| t > 0)
|
||||
.collect();
|
||||
if timestamps.len() < 10 {
|
||||
return None;
|
||||
}
|
||||
timestamps.sort_unstable();
|
||||
|
||||
// Compute inter-arrival deltas
|
||||
let mut deltas = Vec::with_capacity(timestamps.len() - 1);
|
||||
for w in timestamps.windows(2) {
|
||||
let delta = w[1].saturating_sub(w[0]);
|
||||
if delta > 0 {
|
||||
deltas.push(delta as u64);
|
||||
}
|
||||
}
|
||||
if deltas.len() < 5 {
|
||||
return None;
|
||||
}
|
||||
|
||||
// Mean
|
||||
let sum: u64 = deltas.iter().sum();
|
||||
let mean = sum / deltas.len() as u64;
|
||||
if mean == 0 {
|
||||
return None;
|
||||
}
|
||||
|
||||
// Variance (integer arithmetic)
|
||||
let var_sum: u64 = deltas.iter()
|
||||
.map(|&d| {
|
||||
let diff = d.abs_diff(mean);
|
||||
diff * diff
|
||||
})
|
||||
.sum();
|
||||
let variance = var_sum / deltas.len() as u64;
|
||||
|
||||
// Approximate sqrt via integer Newton's method
|
||||
let stddev = isqrt(variance);
|
||||
|
||||
// Coefficient of variation × 1000
|
||||
Some((stddev * 1000 / mean) as u32)
|
||||
}
|
||||
|
||||
/// Detect slow scanning: few unique ports over a long time window.
|
||||
/// Returns true if pattern matches (≤ 1 port/min sustained over 10+ minutes).
|
||||
pub fn is_slow_scanning(&self) -> bool {
|
||||
let age_secs = self.age().as_secs();
|
||||
if age_secs < 600 {
|
||||
// Need at least 10 minutes of observation
|
||||
return false;
|
||||
}
|
||||
let ports = self.port_diversity();
|
||||
if ports < 3 {
|
||||
return false;
|
||||
}
|
||||
// Rate: ports per minute
|
||||
let age_mins = age_secs / 60;
|
||||
if age_mins == 0 {
|
||||
return false;
|
||||
}
|
||||
let ports_per_min = ports as u64 * 100 / age_mins; // ×100 for precision
|
||||
// ≤ 1.5 ports/min = slow scan (150 in ×100 scale)
|
||||
ports_per_min <= 150 && ports >= 5
|
||||
}
|
||||
|
||||
/// Advance to a more suspicious phase (never demote except to Trusted).
|
||||
pub fn escalate_to(&mut self, new_phase: BehaviorPhase) {
|
||||
if new_phase.suspicion_level() > self.phase.suspicion_level() {
|
||||
self.phase = new_phase;
|
||||
self.escalation_count += 1;
|
||||
}
|
||||
}
|
||||
|
||||
/// Promote to Trusted after sustained benign behavior.
|
||||
pub fn promote_to_trusted(&mut self) {
|
||||
self.phase = BehaviorPhase::Trusted;
|
||||
self.suspicion_score = 0.0;
|
||||
}
|
||||
}
|
||||
|
||||
/// Integer square root via Newton's method.
|
||||
fn isqrt(n: u64) -> u64 {
|
||||
if n == 0 {
|
||||
return 0;
|
||||
}
|
||||
let mut x = n;
|
||||
let mut y = x.div_ceil(2);
|
||||
while y < x {
|
||||
x = y;
|
||||
y = (x + n / x) / 2;
|
||||
}
|
||||
x
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn make_event(flags: u8, dst_port: u16, entropy: u32) -> common::PacketEvent {
|
||||
common::PacketEvent {
|
||||
src_ip: 0x0100007f, // 127.0.0.1
|
||||
dst_ip: 0x0200007f,
|
||||
src_port: 12345,
|
||||
dst_port,
|
||||
protocol: 6,
|
||||
flags,
|
||||
payload_len: 64,
|
||||
entropy_score: entropy,
|
||||
timestamp_ns: 0,
|
||||
_padding: 0,
|
||||
packet_size: 128,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn new_profile_defaults() {
|
||||
let p = BehaviorProfile::new();
|
||||
assert_eq!(p.phase, BehaviorPhase::New);
|
||||
assert_eq!(p.total_packets, 0);
|
||||
assert_eq!(p.suspicion_score, 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn update_increments_counters() {
|
||||
let mut p = BehaviorProfile::new();
|
||||
let syn_event = make_event(0x02, 80, 3000);
|
||||
p.update(&syn_event);
|
||||
assert_eq!(p.total_packets, 1);
|
||||
assert_eq!(p.syn_count, 1);
|
||||
assert_eq!(p.ack_count, 0);
|
||||
assert_eq!(p.unique_dst_ports.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn avg_entropy_calculation() {
|
||||
let mut p = BehaviorProfile::new();
|
||||
p.update(&make_event(0x02, 80, 6000));
|
||||
p.update(&make_event(0x02, 443, 8000));
|
||||
assert_eq!(p.avg_entropy(), 7000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn syn_only_ratio() {
|
||||
let mut p = BehaviorProfile::new();
|
||||
// 3 SYN-only
|
||||
for port in 1..=3 {
|
||||
p.update(&make_event(0x02, port, 0));
|
||||
}
|
||||
// 1 SYN+ACK
|
||||
p.update(&make_event(0x12, 80, 0));
|
||||
// syn_count=4, ack_count=1, total=4
|
||||
// syn_only = 4-1 = 3, ratio = 3/4 = 0.75
|
||||
assert!((p.syn_only_ratio() - 0.75).abs() < 0.01);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn escalation_monotonic() {
|
||||
let mut p = BehaviorProfile::new();
|
||||
p.escalate_to(BehaviorPhase::Probing);
|
||||
assert_eq!(p.phase, BehaviorPhase::Probing);
|
||||
// Cannot go back to New
|
||||
p.escalate_to(BehaviorPhase::New);
|
||||
assert_eq!(p.phase, BehaviorPhase::Probing);
|
||||
// Can go forward to Scanning
|
||||
p.escalate_to(BehaviorPhase::Scanning);
|
||||
assert_eq!(p.phase, BehaviorPhase::Scanning);
|
||||
assert_eq!(p.escalation_count, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn trusted_promotion() {
|
||||
let mut p = BehaviorProfile::new();
|
||||
p.escalate_to(BehaviorPhase::Normal);
|
||||
p.suspicion_score = 0.3;
|
||||
p.promote_to_trusted();
|
||||
assert_eq!(p.phase, BehaviorPhase::Trusted);
|
||||
assert_eq!(p.suspicion_score, 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn phase_suspicion_ordering() {
|
||||
assert!(BehaviorPhase::Trusted.suspicion_level() < BehaviorPhase::Normal.suspicion_level());
|
||||
assert!(BehaviorPhase::Normal.suspicion_level() < BehaviorPhase::Probing.suspicion_level());
|
||||
assert!(
|
||||
BehaviorPhase::Probing.suspicion_level() < BehaviorPhase::Scanning.suspicion_level()
|
||||
);
|
||||
assert!(
|
||||
BehaviorPhase::Scanning.suspicion_level()
|
||||
< BehaviorPhase::Exploiting.suspicion_level()
|
||||
);
|
||||
assert!(
|
||||
BehaviorPhase::Exploiting.suspicion_level()
|
||||
< BehaviorPhase::EstablishedC2.suspicion_level()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn actionable_phases() {
|
||||
assert!(!BehaviorPhase::New.is_actionable());
|
||||
assert!(!BehaviorPhase::Normal.is_actionable());
|
||||
assert!(!BehaviorPhase::Trusted.is_actionable());
|
||||
assert!(!BehaviorPhase::Probing.is_actionable());
|
||||
assert!(BehaviorPhase::Scanning.is_actionable());
|
||||
assert!(BehaviorPhase::Exploiting.is_actionable());
|
||||
assert!(BehaviorPhase::EstablishedC2.is_actionable());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn beaconing_insufficient_data() {
|
||||
let p = BehaviorProfile::new();
|
||||
assert!(p.beaconing_score().is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn beaconing_regular_intervals() {
|
||||
let mut p = BehaviorProfile::new();
|
||||
// Simulate 30 packets with regular timestamps (every 1000ns)
|
||||
for i in 0..30 {
|
||||
let mut e = make_event(0x12, 443, 2000);
|
||||
e.timestamp_ns = (i + 1) * 1000;
|
||||
p.update(&e);
|
||||
}
|
||||
if let Some(cv) = p.beaconing_score() {
|
||||
// Regular intervals → low coefficient of variation
|
||||
assert!(cv < 300, "expected low CV for regular beaconing, got {}", cv);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn slow_scan_detection() {
|
||||
let mut p = BehaviorProfile::new();
|
||||
// Simulate slow scanning: spread ports over time
|
||||
// We can't easily fake elapsed time, but we can set up the port diversity
|
||||
for port in 1..=10 {
|
||||
p.update(&make_event(0x02, port, 0));
|
||||
}
|
||||
// With 10 ports and <10min age, should NOT detect
|
||||
assert!(!p.is_slow_scanning());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn isqrt_values() {
|
||||
assert_eq!(super::isqrt(0), 0);
|
||||
assert_eq!(super::isqrt(1), 1);
|
||||
assert_eq!(super::isqrt(4), 2);
|
||||
assert_eq!(super::isqrt(9), 3);
|
||||
assert_eq!(super::isqrt(100), 10);
|
||||
assert_eq!(super::isqrt(1000000), 1000);
|
||||
}
|
||||
}
|
||||
337
blackwall/src/behavior/transitions.rs
Executable file
337
blackwall/src/behavior/transitions.rs
Executable file
|
|
@ -0,0 +1,337 @@
|
|||
//! Deterministic state transitions for the behavioral engine.
|
||||
//!
|
||||
//! Evaluates a `BehaviorProfile` against threshold-based rules and returns
|
||||
//! a `TransitionVerdict` indicating whether to escalate, hold, or promote.
|
||||
|
||||
use super::profile::{BehaviorPhase, BehaviorProfile};
|
||||
|
||||
// --- Transition thresholds ---
|
||||
|
||||
/// Unique destination ports to trigger Probing escalation.
|
||||
const PROBING_PORT_THRESHOLD: usize = 5;
|
||||
/// Unique destination ports to trigger Scanning escalation.
|
||||
const SCANNING_PORT_THRESHOLD: usize = 20;
|
||||
/// SYN-only ratio above this → likely SYN flood or scan.
|
||||
const SYN_FLOOD_RATIO: f32 = 0.8;
|
||||
/// Minimum packets for SYN flood detection.
|
||||
const SYN_FLOOD_MIN_PACKETS: u64 = 50;
|
||||
/// Average byte diversity score above this → encrypted/exploit payload.
|
||||
const EXPLOIT_ENTROPY_THRESHOLD: u32 = 7500;
|
||||
/// Minimum entropy samples for exploit detection.
|
||||
const EXPLOIT_MIN_SAMPLES: u64 = 10;
|
||||
/// RST ratio above this (with sufficient packets) → scanning/exploit.
|
||||
const RST_RATIO_THRESHOLD: f32 = 0.5;
|
||||
/// Minimum packets to evaluate RST ratio.
|
||||
const RST_RATIO_MIN_PACKETS: u64 = 20;
|
||||
/// Beaconing coefficient of variation threshold (×1000).
|
||||
/// Values below this indicate highly regular intervals (C2 beaconing).
|
||||
const BEACONING_CV_THRESHOLD: u32 = 300;
|
||||
/// Packets needed to promote New → Normal.
|
||||
const NORMAL_PACKET_THRESHOLD: u64 = 10;
|
||||
/// Seconds of benign activity before promoting Normal → Trusted.
|
||||
const TRUSTED_AGE_SECS: u64 = 300;
|
||||
/// Minimum packets for Trusted promotion.
|
||||
const TRUSTED_PACKET_THRESHOLD: u64 = 100;
|
||||
/// Suspicion score increase per escalation event.
|
||||
const SUSPICION_INCREMENT: f32 = 0.15;
|
||||
/// Maximum suspicion score.
|
||||
const SUSPICION_MAX: f32 = 1.0;
|
||||
/// Suspicion decay per evaluation when no escalation.
|
||||
const SUSPICION_DECAY: f32 = 0.02;
|
||||
|
||||
/// Result of evaluating a profile's behavioral transitions.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum TransitionVerdict {
|
||||
/// No phase change, continue monitoring.
|
||||
Hold,
|
||||
/// Escalate to a more suspicious phase.
|
||||
Escalate {
|
||||
from: BehaviorPhase,
|
||||
to: BehaviorPhase,
|
||||
reason: &'static str,
|
||||
},
|
||||
/// Promote to a less suspicious phase (Trusted).
|
||||
Promote {
|
||||
from: BehaviorPhase,
|
||||
to: BehaviorPhase,
|
||||
},
|
||||
}
|
||||
|
||||
/// Evaluate a profile and apply deterministic transitions.
|
||||
/// Returns the verdict and mutates the profile in place.
|
||||
pub fn evaluate_transitions(profile: &mut BehaviorProfile) -> TransitionVerdict {
|
||||
if !profile.has_sufficient_data() {
|
||||
return TransitionVerdict::Hold;
|
||||
}
|
||||
|
||||
let current = profile.phase;
|
||||
|
||||
// --- Check for escalation conditions (highest severity first) ---
|
||||
|
||||
// C2 beaconing: sustained high entropy with regular intervals + many packets
|
||||
if current.suspicion_level() < BehaviorPhase::EstablishedC2.suspicion_level()
|
||||
&& profile.avg_entropy() > EXPLOIT_ENTROPY_THRESHOLD
|
||||
&& profile.total_packets > 200
|
||||
&& profile.port_diversity() <= 3
|
||||
&& profile.age().as_secs() > 60
|
||||
{
|
||||
return apply_escalation(
|
||||
profile,
|
||||
BehaviorPhase::EstablishedC2,
|
||||
"sustained high entropy with low port diversity (C2 pattern)",
|
||||
);
|
||||
}
|
||||
|
||||
// C2 beaconing: regular inter-arrival intervals (even without high entropy)
|
||||
if current.suspicion_level() < BehaviorPhase::EstablishedC2.suspicion_level()
|
||||
&& profile.total_packets > 100
|
||||
&& profile.age().as_secs() > 120
|
||||
{
|
||||
if let Some(cv) = profile.beaconing_score() {
|
||||
if cv < BEACONING_CV_THRESHOLD && profile.port_diversity() <= 3 {
|
||||
return apply_escalation(
|
||||
profile,
|
||||
BehaviorPhase::EstablishedC2,
|
||||
"regular beaconing intervals detected (C2 callback pattern)",
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Exploit: high entropy payloads
|
||||
if current.suspicion_level() < BehaviorPhase::Exploiting.suspicion_level()
|
||||
&& profile.avg_entropy() > EXPLOIT_ENTROPY_THRESHOLD
|
||||
&& profile.entropy_samples >= EXPLOIT_MIN_SAMPLES
|
||||
{
|
||||
return apply_escalation(
|
||||
profile,
|
||||
BehaviorPhase::Exploiting,
|
||||
"high entropy payloads (encrypted/exploit traffic)",
|
||||
);
|
||||
}
|
||||
|
||||
// Scanning: many unique ports
|
||||
if current.suspicion_level() < BehaviorPhase::Scanning.suspicion_level()
|
||||
&& profile.port_diversity() > SCANNING_PORT_THRESHOLD
|
||||
{
|
||||
return apply_escalation(
|
||||
profile,
|
||||
BehaviorPhase::Scanning,
|
||||
"extensive port scanning (>20 unique ports)",
|
||||
);
|
||||
}
|
||||
|
||||
// SYN flood: high SYN-only ratio with sufficient volume
|
||||
if current.suspicion_level() < BehaviorPhase::Scanning.suspicion_level()
|
||||
&& profile.syn_only_ratio() > SYN_FLOOD_RATIO
|
||||
&& profile.total_packets >= SYN_FLOOD_MIN_PACKETS
|
||||
{
|
||||
return apply_escalation(
|
||||
profile,
|
||||
BehaviorPhase::Scanning,
|
||||
"SYN flood pattern (>80% SYN-only, >50 packets)",
|
||||
);
|
||||
}
|
||||
|
||||
// Slow scan: few ports over a long time window (stealth reconnaissance)
|
||||
if current.suspicion_level() < BehaviorPhase::Scanning.suspicion_level()
|
||||
&& profile.is_slow_scanning()
|
||||
{
|
||||
return apply_escalation(
|
||||
profile,
|
||||
BehaviorPhase::Scanning,
|
||||
"slow scan pattern (≤1.5 ports/min over 10+ minutes)",
|
||||
);
|
||||
}
|
||||
|
||||
// RST storm: many connection resets (scanner getting rejected)
|
||||
if current.suspicion_level() < BehaviorPhase::Probing.suspicion_level()
|
||||
&& profile.total_packets >= RST_RATIO_MIN_PACKETS
|
||||
{
|
||||
let rst_ratio = profile.rst_count as f32 / profile.total_packets as f32;
|
||||
if rst_ratio > RST_RATIO_THRESHOLD {
|
||||
return apply_escalation(
|
||||
profile,
|
||||
BehaviorPhase::Probing,
|
||||
"high RST ratio (>50%, scanning likely rejected)",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Probing: moderate port diversity
|
||||
if current.suspicion_level() < BehaviorPhase::Probing.suspicion_level()
|
||||
&& profile.port_diversity() > PROBING_PORT_THRESHOLD
|
||||
{
|
||||
return apply_escalation(
|
||||
profile,
|
||||
BehaviorPhase::Probing,
|
||||
"port diversity above probing threshold (>5 unique ports)",
|
||||
);
|
||||
}
|
||||
|
||||
// --- Check for promotion conditions ---
|
||||
|
||||
// New → Normal: sufficient packets without triggering any escalation
|
||||
if current == BehaviorPhase::New && profile.total_packets >= NORMAL_PACKET_THRESHOLD {
|
||||
profile.phase = BehaviorPhase::Normal;
|
||||
return TransitionVerdict::Promote {
|
||||
from: BehaviorPhase::New,
|
||||
to: BehaviorPhase::Normal,
|
||||
};
|
||||
}
|
||||
|
||||
// Normal → Trusted: sustained benign behavior
|
||||
if current == BehaviorPhase::Normal
|
||||
&& profile.age().as_secs() >= TRUSTED_AGE_SECS
|
||||
&& profile.total_packets >= TRUSTED_PACKET_THRESHOLD
|
||||
&& profile.suspicion_score < 0.1
|
||||
{
|
||||
profile.promote_to_trusted();
|
||||
return TransitionVerdict::Promote {
|
||||
from: BehaviorPhase::Normal,
|
||||
to: BehaviorPhase::Trusted,
|
||||
};
|
||||
}
|
||||
|
||||
// --- No transition: decay suspicion slightly ---
|
||||
profile.suspicion_score = (profile.suspicion_score - SUSPICION_DECAY).max(0.0);
|
||||
|
||||
TransitionVerdict::Hold
|
||||
}
|
||||
|
||||
/// Apply an escalation: update phase, bump suspicion, return verdict.
|
||||
fn apply_escalation(
|
||||
profile: &mut BehaviorProfile,
|
||||
target: BehaviorPhase,
|
||||
reason: &'static str,
|
||||
) -> TransitionVerdict {
|
||||
let from = profile.phase;
|
||||
profile.escalate_to(target);
|
||||
profile.suspicion_score = (profile.suspicion_score + SUSPICION_INCREMENT).min(SUSPICION_MAX);
|
||||
TransitionVerdict::Escalate {
|
||||
from,
|
||||
to: target,
|
||||
reason,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn make_event(flags: u8, dst_port: u16, entropy: u32) -> common::PacketEvent {
|
||||
common::PacketEvent {
|
||||
src_ip: 0x0100007f,
|
||||
dst_ip: 0x0200007f,
|
||||
src_port: 12345,
|
||||
dst_port,
|
||||
protocol: 6,
|
||||
flags,
|
||||
payload_len: 64,
|
||||
entropy_score: entropy,
|
||||
timestamp_ns: 0,
|
||||
_padding: 0,
|
||||
packet_size: 128,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn insufficient_data_holds() {
|
||||
let mut p = BehaviorProfile::new();
|
||||
p.update(&make_event(0x02, 80, 3000));
|
||||
assert_eq!(evaluate_transitions(&mut p), TransitionVerdict::Hold);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn new_to_normal_promotion() {
|
||||
let mut p = BehaviorProfile::new();
|
||||
// 10 benign packets to same port
|
||||
for _ in 0..10 {
|
||||
p.update(&make_event(0x12, 80, 2000)); // SYN+ACK
|
||||
}
|
||||
let v = evaluate_transitions(&mut p);
|
||||
assert_eq!(
|
||||
v,
|
||||
TransitionVerdict::Promote {
|
||||
from: BehaviorPhase::New,
|
||||
to: BehaviorPhase::Normal,
|
||||
}
|
||||
);
|
||||
assert_eq!(p.phase, BehaviorPhase::Normal);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn port_scan_escalation() {
|
||||
let mut p = BehaviorProfile::new();
|
||||
// Hit 25 unique ports (> SCANNING_PORT_THRESHOLD=20)
|
||||
for port in 1..=25 {
|
||||
p.update(&make_event(0x02, port, 1000));
|
||||
}
|
||||
let v = evaluate_transitions(&mut p);
|
||||
match v {
|
||||
TransitionVerdict::Escalate { to, reason, .. } => {
|
||||
assert_eq!(to, BehaviorPhase::Scanning);
|
||||
assert!(reason.contains("port scanning"));
|
||||
}
|
||||
other => panic!("expected Scanning escalation, got {:?}", other),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn syn_flood_escalation() {
|
||||
let mut p = BehaviorProfile::new();
|
||||
// 60 SYN-only packets to same port
|
||||
for _ in 0..60 {
|
||||
p.update(&make_event(0x02, 80, 0));
|
||||
}
|
||||
let v = evaluate_transitions(&mut p);
|
||||
match v {
|
||||
TransitionVerdict::Escalate { to, reason, .. } => {
|
||||
assert_eq!(to, BehaviorPhase::Scanning);
|
||||
assert!(reason.contains("SYN flood"));
|
||||
}
|
||||
other => panic!("expected SYN flood escalation, got {:?}", other),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn high_entropy_exploit() {
|
||||
let mut p = BehaviorProfile::new();
|
||||
// 15 high-entropy packets
|
||||
for port in 1..=15 {
|
||||
p.update(&make_event(0x12, port, 7800));
|
||||
}
|
||||
let v = evaluate_transitions(&mut p);
|
||||
match v {
|
||||
TransitionVerdict::Escalate { to, reason, .. } => {
|
||||
assert_eq!(to, BehaviorPhase::Exploiting);
|
||||
assert!(reason.contains("entropy"));
|
||||
}
|
||||
other => panic!("expected Exploiting escalation, got {:?}", other),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn suspicion_increases_on_escalation() {
|
||||
let mut p = BehaviorProfile::new();
|
||||
assert_eq!(p.suspicion_score, 0.0);
|
||||
for port in 1..=6 {
|
||||
p.update(&make_event(0x02, port, 1000));
|
||||
}
|
||||
evaluate_transitions(&mut p);
|
||||
assert!(p.suspicion_score > 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn suspicion_decays_when_benign() {
|
||||
let mut p = BehaviorProfile::new();
|
||||
p.suspicion_score = 0.5;
|
||||
p.phase = BehaviorPhase::Normal;
|
||||
// Feed benign traffic (same port, low entropy)
|
||||
for _ in 0..10 {
|
||||
p.update(&make_event(0x12, 80, 2000));
|
||||
}
|
||||
evaluate_transitions(&mut p);
|
||||
assert!(p.suspicion_score < 0.5);
|
||||
}
|
||||
}
|
||||
367
blackwall/src/config.rs
Executable file
367
blackwall/src/config.rs
Executable file
|
|
@ -0,0 +1,367 @@
|
|||
use serde::Deserialize;
|
||||
use std::path::Path;
|
||||
|
||||
/// Top-level daemon configuration, loaded from TOML.
|
||||
#[derive(Deserialize)]
|
||||
pub struct Config {
|
||||
pub network: NetworkConfig,
|
||||
#[serde(default)]
|
||||
#[allow(dead_code)]
|
||||
pub thresholds: ThresholdConfig,
|
||||
#[serde(default)]
|
||||
pub tarpit: TarpitConfig,
|
||||
#[serde(default)]
|
||||
pub ai: AiConfig,
|
||||
#[serde(default)]
|
||||
pub rules: RulesConfig,
|
||||
#[serde(default)]
|
||||
pub feeds: FeedsConfig,
|
||||
#[serde(default)]
|
||||
pub pcap: PcapConfig,
|
||||
#[serde(default)]
|
||||
#[allow(dead_code)]
|
||||
pub distributed: DistributedConfig,
|
||||
}
|
||||
|
||||
/// Network / XDP attachment settings.
|
||||
#[derive(Deserialize)]
|
||||
pub struct NetworkConfig {
|
||||
/// Network interface to attach XDP program to.
|
||||
#[serde(default = "default_interface")]
|
||||
pub interface: String,
|
||||
/// XDP attach mode: "generic", "native", or "offload".
|
||||
#[serde(default = "default_xdp_mode")]
|
||||
pub xdp_mode: String,
|
||||
}
|
||||
|
||||
/// Anomaly detection thresholds.
|
||||
#[derive(Deserialize)]
|
||||
#[allow(dead_code)]
|
||||
pub struct ThresholdConfig {
|
||||
/// Byte diversity score above which a packet is considered anomalous.
|
||||
#[serde(default = "default_entropy_anomaly")]
|
||||
pub entropy_anomaly: u32,
|
||||
}
|
||||
|
||||
/// Tarpit honeypot configuration.
|
||||
#[derive(Deserialize)]
|
||||
#[allow(dead_code)]
|
||||
pub struct TarpitConfig {
|
||||
#[serde(default = "default_true")]
|
||||
pub enabled: bool,
|
||||
#[serde(default = "default_tarpit_port")]
|
||||
pub port: u16,
|
||||
#[serde(default = "default_base_delay")]
|
||||
pub base_delay_ms: u64,
|
||||
#[serde(default = "default_max_delay")]
|
||||
pub max_delay_ms: u64,
|
||||
#[serde(default = "default_jitter")]
|
||||
pub jitter_ms: u64,
|
||||
/// Per-protocol deception service port overrides.
|
||||
#[serde(default)]
|
||||
pub services: DeceptionServicesConfig,
|
||||
}
|
||||
|
||||
/// Per-protocol port configuration for the deception mesh.
|
||||
#[derive(Deserialize)]
|
||||
#[allow(dead_code)]
|
||||
pub struct DeceptionServicesConfig {
|
||||
/// SSH honeypot port (default: 22).
|
||||
#[serde(default = "default_ssh_port")]
|
||||
pub ssh_port: u16,
|
||||
/// HTTP honeypot port (default: 80).
|
||||
#[serde(default = "default_http_port")]
|
||||
pub http_port: u16,
|
||||
/// MySQL honeypot port (default: 3306).
|
||||
#[serde(default = "default_mysql_port")]
|
||||
pub mysql_port: u16,
|
||||
/// DNS canary port (default: 53).
|
||||
#[serde(default = "default_dns_port")]
|
||||
pub dns_port: u16,
|
||||
}
|
||||
|
||||
/// AI / LLM classification settings.
|
||||
#[derive(Deserialize)]
|
||||
#[allow(dead_code)]
|
||||
pub struct AiConfig {
|
||||
#[serde(default = "default_true")]
|
||||
pub enabled: bool,
|
||||
#[serde(default = "default_ollama_url")]
|
||||
pub ollama_url: String,
|
||||
#[serde(default = "default_model")]
|
||||
pub model: String,
|
||||
#[serde(default = "default_fallback_model")]
|
||||
pub fallback_model: String,
|
||||
#[serde(default = "default_max_tokens")]
|
||||
pub max_tokens: u32,
|
||||
#[serde(default = "default_timeout_ms")]
|
||||
pub timeout_ms: u64,
|
||||
}
|
||||
|
||||
/// Static rules loaded at startup.
|
||||
#[derive(Deserialize, Default)]
|
||||
pub struct RulesConfig {
|
||||
#[serde(default)]
|
||||
pub blocklist: Vec<String>,
|
||||
#[serde(default)]
|
||||
pub allowlist: Vec<String>,
|
||||
}
|
||||
|
||||
/// Threat feed configuration.
|
||||
#[derive(Deserialize)]
|
||||
pub struct FeedsConfig {
|
||||
/// Whether threat feed fetching is enabled.
|
||||
#[serde(default = "default_true")]
|
||||
pub enabled: bool,
|
||||
/// Refresh interval in seconds (default: 1 hour).
|
||||
#[serde(default = "default_feed_refresh_secs")]
|
||||
pub refresh_interval_secs: u64,
|
||||
/// Block duration for feed-sourced IPs in seconds (default: 1 hour).
|
||||
#[serde(default = "default_feed_block_secs")]
|
||||
pub block_duration_secs: u32,
|
||||
/// Feed source URLs.
|
||||
#[serde(default = "default_feed_sources")]
|
||||
pub sources: Vec<FeedSourceConfig>,
|
||||
}
|
||||
|
||||
/// A single threat feed source entry.
|
||||
#[derive(Deserialize, Clone)]
|
||||
pub struct FeedSourceConfig {
|
||||
pub name: String,
|
||||
pub url: String,
|
||||
/// Override block duration for this feed (uses parent default if absent).
|
||||
pub block_duration_secs: Option<u32>,
|
||||
}
|
||||
|
||||
/// PCAP forensic capture configuration.
|
||||
#[derive(Deserialize)]
|
||||
#[allow(dead_code)]
|
||||
pub struct PcapConfig {
|
||||
/// Whether PCAP capture is enabled.
|
||||
#[serde(default)]
|
||||
pub enabled: bool,
|
||||
/// Output directory for pcap files.
|
||||
#[serde(default = "default_pcap_dir")]
|
||||
pub output_dir: String,
|
||||
/// Maximum pcap file size in MB before rotation.
|
||||
#[serde(default = "default_pcap_max_size")]
|
||||
pub max_size_mb: u64,
|
||||
/// Maximum number of rotated pcap files to keep.
|
||||
#[serde(default = "default_pcap_max_files")]
|
||||
pub max_files: usize,
|
||||
/// Compress rotated pcap files with gzip.
|
||||
#[serde(default)]
|
||||
pub compress_rotated: bool,
|
||||
}
|
||||
|
||||
/// Distributed coordination configuration.
|
||||
#[derive(Deserialize)]
|
||||
#[allow(dead_code)]
|
||||
pub struct DistributedConfig {
|
||||
/// Whether distributed mode is enabled.
|
||||
#[serde(default)]
|
||||
pub enabled: bool,
|
||||
/// Mode: "sensor" (reports to controller) or "standalone" (default).
|
||||
#[serde(default = "default_distributed_mode")]
|
||||
pub mode: String,
|
||||
/// Peer addresses to connect to.
|
||||
#[serde(default)]
|
||||
pub peers: Vec<String>,
|
||||
/// Port to listen for peer connections.
|
||||
#[serde(default = "default_peer_port")]
|
||||
pub bind_port: u16,
|
||||
/// Node identifier (auto-generated if empty).
|
||||
#[serde(default)]
|
||||
pub node_id: String,
|
||||
/// Pre-shared key for HMAC-SHA256 peer authentication.
|
||||
/// All peers in the mesh must share the same PSK.
|
||||
/// If empty, distributed mode refuses to start.
|
||||
#[serde(default)]
|
||||
pub peer_psk: String,
|
||||
}
|
||||
|
||||
// --- Defaults ---
|
||||
|
||||
fn default_interface() -> String {
|
||||
"eth0".into()
|
||||
}
|
||||
fn default_xdp_mode() -> String {
|
||||
"generic".into()
|
||||
}
|
||||
fn default_entropy_anomaly() -> u32 {
|
||||
common::ENTROPY_ANOMALY_THRESHOLD
|
||||
}
|
||||
fn default_true() -> bool {
|
||||
true
|
||||
}
|
||||
fn default_tarpit_port() -> u16 {
|
||||
common::TARPIT_PORT
|
||||
}
|
||||
fn default_base_delay() -> u64 {
|
||||
common::TARPIT_BASE_DELAY_MS
|
||||
}
|
||||
fn default_max_delay() -> u64 {
|
||||
common::TARPIT_MAX_DELAY_MS
|
||||
}
|
||||
fn default_jitter() -> u64 {
|
||||
common::TARPIT_JITTER_MS
|
||||
}
|
||||
fn default_ollama_url() -> String {
|
||||
"http://localhost:11434".into()
|
||||
}
|
||||
fn default_model() -> String {
|
||||
"qwen3:1.7b".into()
|
||||
}
|
||||
fn default_fallback_model() -> String {
|
||||
"qwen3:0.6b".into()
|
||||
}
|
||||
fn default_max_tokens() -> u32 {
|
||||
512
|
||||
}
|
||||
fn default_timeout_ms() -> u64 {
|
||||
5000
|
||||
}
|
||||
fn default_feed_refresh_secs() -> u64 {
|
||||
3600
|
||||
}
|
||||
fn default_feed_block_secs() -> u32 {
|
||||
3600
|
||||
}
|
||||
fn default_feed_sources() -> Vec<FeedSourceConfig> {
|
||||
vec![
|
||||
FeedSourceConfig {
|
||||
name: "firehol-level1".into(),
|
||||
url: "https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level1.netset".into(),
|
||||
block_duration_secs: None,
|
||||
},
|
||||
FeedSourceConfig {
|
||||
name: "feodo-tracker".into(),
|
||||
url: "https://feodotracker.abuse.ch/downloads/ipblocklist.txt".into(),
|
||||
block_duration_secs: None,
|
||||
},
|
||||
]
|
||||
}
|
||||
fn default_pcap_dir() -> String {
|
||||
"/var/lib/blackwall/pcap".into()
|
||||
}
|
||||
fn default_pcap_max_size() -> u64 {
|
||||
100
|
||||
}
|
||||
fn default_pcap_max_files() -> usize {
|
||||
10
|
||||
}
|
||||
fn default_ssh_port() -> u16 {
|
||||
22
|
||||
}
|
||||
fn default_http_port() -> u16 {
|
||||
80
|
||||
}
|
||||
fn default_mysql_port() -> u16 {
|
||||
3306
|
||||
}
|
||||
fn default_dns_port() -> u16 {
|
||||
53
|
||||
}
|
||||
fn default_distributed_mode() -> String {
|
||||
"standalone".into()
|
||||
}
|
||||
fn default_peer_port() -> u16 {
|
||||
9471
|
||||
}
|
||||
|
||||
impl Default for NetworkConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
interface: default_interface(),
|
||||
xdp_mode: default_xdp_mode(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for ThresholdConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
entropy_anomaly: default_entropy_anomaly(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for TarpitConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
enabled: true,
|
||||
port: default_tarpit_port(),
|
||||
base_delay_ms: default_base_delay(),
|
||||
max_delay_ms: default_max_delay(),
|
||||
jitter_ms: default_jitter(),
|
||||
services: DeceptionServicesConfig::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for DeceptionServicesConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
ssh_port: default_ssh_port(),
|
||||
http_port: default_http_port(),
|
||||
mysql_port: default_mysql_port(),
|
||||
dns_port: default_dns_port(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for AiConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
enabled: true,
|
||||
ollama_url: default_ollama_url(),
|
||||
model: default_model(),
|
||||
fallback_model: default_fallback_model(),
|
||||
max_tokens: default_max_tokens(),
|
||||
timeout_ms: default_timeout_ms(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for FeedsConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
enabled: true,
|
||||
refresh_interval_secs: default_feed_refresh_secs(),
|
||||
block_duration_secs: default_feed_block_secs(),
|
||||
sources: default_feed_sources(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for PcapConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
enabled: false,
|
||||
output_dir: default_pcap_dir(),
|
||||
max_size_mb: default_pcap_max_size(),
|
||||
max_files: default_pcap_max_files(),
|
||||
compress_rotated: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for DistributedConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
enabled: false,
|
||||
mode: default_distributed_mode(),
|
||||
peers: Vec::new(),
|
||||
bind_port: default_peer_port(),
|
||||
node_id: String::new(),
|
||||
peer_psk: String::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Load configuration from a TOML file.
|
||||
pub fn load_config(path: &Path) -> anyhow::Result<Config> {
|
||||
let content = std::fs::read_to_string(path)?;
|
||||
let config: Config = toml::from_str(&content)?;
|
||||
Ok(config)
|
||||
}
|
||||
11
blackwall/src/distributed/mod.rs
Executable file
11
blackwall/src/distributed/mod.rs
Executable file
|
|
@ -0,0 +1,11 @@
|
|||
//! Distributed coordination module for multi-instance Blackwall deployments.
|
||||
//!
|
||||
//! Enables multiple Blackwall nodes to share threat intelligence via a
|
||||
//! simple peer-to-peer protocol over TCP. Nodes exchange blocked IPs,
|
||||
//! JA4 fingerprints, and behavioral observations.
|
||||
|
||||
pub mod peer;
|
||||
pub mod proto;
|
||||
|
||||
#[allow(unused_imports)]
|
||||
pub use peer::{broadcast_block, PeerManager};
|
||||
500
blackwall/src/distributed/peer.rs
Executable file
500
blackwall/src/distributed/peer.rs
Executable file
|
|
@ -0,0 +1,500 @@
|
|||
//! Peer management: discovery, connection, and message exchange.
|
||||
//!
|
||||
//! Manages connections to other Blackwall nodes for distributed
|
||||
//! threat intelligence sharing.
|
||||
#![allow(dead_code)]
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use ring::hmac;
|
||||
use std::collections::HashMap;
|
||||
use std::net::{Ipv4Addr, SocketAddr};
|
||||
use std::time::{Duration, Instant};
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use tokio::net::{TcpListener, TcpStream};
|
||||
|
||||
use super::proto::{self, BlockedIpPayload, HelloPayload, MessageType, HEADER_SIZE};
|
||||
|
||||
/// Default port for peer communication.
|
||||
pub const DEFAULT_PEER_PORT: u16 = 9471;
|
||||
/// Heartbeat interval.
|
||||
const HEARTBEAT_INTERVAL: Duration = Duration::from_secs(30);
|
||||
/// Peer connection timeout.
|
||||
const CONNECT_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
/// Maximum peers to maintain.
|
||||
const MAX_PEERS: usize = 16;
|
||||
/// Maximum message payload size (64 KB).
|
||||
const MAX_PAYLOAD_SIZE: usize = 65536;
|
||||
|
||||
/// Known peer state.
|
||||
struct PeerState {
|
||||
addr: SocketAddr,
|
||||
node_id: Option<String>,
|
||||
last_seen: Instant,
|
||||
blocked_count: u32,
|
||||
/// Persistent outbound connection for broadcasts (reused across sends).
|
||||
outbound: Option<TcpStream>,
|
||||
}
|
||||
|
||||
/// Manages distributed peer connections and threat intel sharing.
|
||||
pub struct PeerManager {
|
||||
/// Our node identifier
|
||||
node_id: String,
|
||||
/// Known peers with their state
|
||||
peers: HashMap<SocketAddr, PeerState>,
|
||||
/// IPs received from peers (ip → source_peer)
|
||||
shared_blocks: HashMap<Ipv4Addr, SocketAddr>,
|
||||
/// HMAC-SHA256 key derived from peer PSK for message authentication.
|
||||
hmac_key: hmac::Key,
|
||||
}
|
||||
|
||||
impl PeerManager {
|
||||
/// Create a new peer manager with the given node ID and pre-shared key.
|
||||
pub fn new(node_id: String, peer_psk: &[u8]) -> Self {
|
||||
Self {
|
||||
node_id,
|
||||
peers: HashMap::new(),
|
||||
shared_blocks: HashMap::new(),
|
||||
hmac_key: hmac::Key::new(hmac::HMAC_SHA256, peer_psk),
|
||||
}
|
||||
}
|
||||
|
||||
/// Get reference to the HMAC key for message signing/verification.
|
||||
pub fn hmac_key(&self) -> &hmac::Key {
|
||||
&self.hmac_key
|
||||
}
|
||||
|
||||
/// Add a peer address to the known peers list.
|
||||
pub fn add_peer(&mut self, addr: SocketAddr) {
|
||||
if self.peers.len() >= MAX_PEERS {
|
||||
tracing::warn!("max peers reached, ignoring {}", addr);
|
||||
return;
|
||||
}
|
||||
self.peers.entry(addr).or_insert_with(|| PeerState {
|
||||
addr,
|
||||
node_id: None,
|
||||
last_seen: Instant::now(),
|
||||
blocked_count: 0,
|
||||
outbound: None,
|
||||
});
|
||||
}
|
||||
|
||||
/// Get count of known peers.
|
||||
pub fn peer_count(&self) -> usize {
|
||||
self.peers.len()
|
||||
}
|
||||
|
||||
/// Get count of shared block entries received from peers.
|
||||
#[allow(dead_code)]
|
||||
pub fn shared_block_count(&self) -> usize {
|
||||
self.shared_blocks.len()
|
||||
}
|
||||
|
||||
/// Process a received blocked IP notification from a peer.
|
||||
pub fn receive_blocked_ip(
|
||||
&mut self,
|
||||
from: SocketAddr,
|
||||
payload: &BlockedIpPayload,
|
||||
) -> Option<(Ipv4Addr, u32)> {
|
||||
// Only accept if confidence is reasonable
|
||||
if payload.confidence < 50 {
|
||||
tracing::debug!(
|
||||
peer = %from,
|
||||
ip = %payload.ip,
|
||||
confidence = payload.confidence,
|
||||
"ignoring low-confidence peer block"
|
||||
);
|
||||
return None;
|
||||
}
|
||||
|
||||
self.shared_blocks.insert(payload.ip, from);
|
||||
|
||||
tracing::info!(
|
||||
peer = %from,
|
||||
ip = %payload.ip,
|
||||
reason = %payload.reason,
|
||||
confidence = payload.confidence,
|
||||
"received blocked IP from peer"
|
||||
);
|
||||
|
||||
Some((payload.ip, payload.duration_secs))
|
||||
}
|
||||
|
||||
/// Create a hello payload for this node.
|
||||
pub fn make_hello(&self, blocked_count: u32) -> HelloPayload {
|
||||
HelloPayload {
|
||||
node_id: self.node_id.clone(),
|
||||
version: env!("CARGO_PKG_VERSION").to_string(),
|
||||
blocked_count,
|
||||
}
|
||||
}
|
||||
|
||||
/// Handle an incoming hello from a peer.
|
||||
pub fn handle_hello(&mut self, from: SocketAddr, hello: &HelloPayload) {
|
||||
if let Some(peer) = self.peers.get_mut(&from) {
|
||||
peer.node_id = Some(hello.node_id.clone());
|
||||
peer.last_seen = Instant::now();
|
||||
peer.blocked_count = hello.blocked_count;
|
||||
}
|
||||
tracing::info!(
|
||||
peer = %from,
|
||||
node_id = %hello.node_id,
|
||||
blocked = hello.blocked_count,
|
||||
"peer hello received"
|
||||
);
|
||||
}
|
||||
|
||||
/// Prune peers that haven't been seen in a while.
|
||||
pub fn prune_stale_peers(&mut self, max_age: Duration) {
|
||||
let before = self.peers.len();
|
||||
self.peers.retain(|_, p| p.last_seen.elapsed() < max_age);
|
||||
let pruned = before - self.peers.len();
|
||||
if pruned > 0 {
|
||||
tracing::info!(count = pruned, "pruned stale peers");
|
||||
}
|
||||
}
|
||||
|
||||
/// Get addresses of all known peers.
|
||||
pub fn peer_addrs(&self) -> Vec<SocketAddr> {
|
||||
self.peers.keys().copied().collect()
|
||||
}
|
||||
|
||||
/// Take outbound streams from all peers for concurrent use.
|
||||
/// Returns (addr, Option<TcpStream>) pairs. Caller MUST return streams
|
||||
/// via `return_outbound()` after use.
|
||||
fn take_outbound_streams(&mut self) -> Vec<(SocketAddr, Option<TcpStream>)> {
|
||||
self.peers
|
||||
.iter_mut()
|
||||
.map(|(addr, state)| (*addr, state.outbound.take()))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Return an outbound stream (or None if it broke) to a peer.
|
||||
fn return_outbound(&mut self, addr: &SocketAddr, stream: Option<TcpStream>) {
|
||||
if let Some(peer) = self.peers.get_mut(addr) {
|
||||
peer.outbound = stream;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Broadcast a blocked IP to all known peers.
|
||||
///
|
||||
/// Reuses persistent outbound TCP connections where possible. Creates new
|
||||
/// connections only when no existing stream is available or the previous
|
||||
/// one has broken. Sends in parallel; individual peer failures are logged
|
||||
/// and do not block other peers.
|
||||
pub async fn broadcast_block(
|
||||
manager: &std::sync::Arc<tokio::sync::Mutex<PeerManager>>,
|
||||
payload: &BlockedIpPayload,
|
||||
) {
|
||||
let peers = {
|
||||
let mut mgr = manager.lock().await;
|
||||
mgr.take_outbound_streams()
|
||||
};
|
||||
|
||||
if peers.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
tracing::info!(
|
||||
ip = %payload.ip,
|
||||
peers = peers.len(),
|
||||
"broadcasting block to peers"
|
||||
);
|
||||
|
||||
let json = match serde_json::to_vec(payload) {
|
||||
Ok(j) => j,
|
||||
Err(e) => {
|
||||
// Return streams untouched on serialization failure
|
||||
let mut mgr = manager.lock().await;
|
||||
for (addr, stream) in peers {
|
||||
mgr.return_outbound(&addr, stream);
|
||||
}
|
||||
tracing::warn!(error = %e, "failed to serialize BlockedIpPayload");
|
||||
return;
|
||||
}
|
||||
};
|
||||
let msg = {
|
||||
let mgr = manager.lock().await;
|
||||
proto::encode_message(MessageType::BlockedIp, &json, mgr.hmac_key())
|
||||
};
|
||||
|
||||
let mut tasks = Vec::with_capacity(peers.len());
|
||||
for (addr, existing) in peers {
|
||||
let msg_clone = msg.clone();
|
||||
tasks.push(tokio::spawn(async move {
|
||||
let stream = send_or_reconnect(addr, existing, &msg_clone).await;
|
||||
(addr, stream)
|
||||
}));
|
||||
}
|
||||
|
||||
// Return streams (live or None) back to the manager
|
||||
let mut mgr = manager.lock().await;
|
||||
for task in tasks {
|
||||
if let Ok((addr, stream)) = task.await {
|
||||
mgr.return_outbound(&addr, stream);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Try to send on an existing connection; reconnect if broken.
|
||||
/// Returns the stream if it's still alive, or None if the peer is unreachable.
|
||||
async fn send_or_reconnect(
|
||||
addr: SocketAddr,
|
||||
existing: Option<TcpStream>,
|
||||
msg: &[u8],
|
||||
) -> Option<TcpStream> {
|
||||
// Try existing connection first
|
||||
if let Some(mut stream) = existing {
|
||||
if stream.write_all(msg).await.is_ok() && stream.flush().await.is_ok() {
|
||||
return Some(stream);
|
||||
}
|
||||
// Connection broken — fall through to reconnect
|
||||
tracing::debug!(peer = %addr, "outbound stream broken, reconnecting");
|
||||
}
|
||||
|
||||
// Create new connection
|
||||
let mut stream = match tokio::time::timeout(CONNECT_TIMEOUT, TcpStream::connect(addr)).await {
|
||||
Ok(Ok(s)) => s,
|
||||
Ok(Err(e)) => {
|
||||
tracing::warn!(peer = %addr, error = %e, "peer connect failed");
|
||||
return None;
|
||||
}
|
||||
Err(_) => {
|
||||
tracing::warn!(peer = %addr, "peer connect timeout");
|
||||
return None;
|
||||
}
|
||||
};
|
||||
|
||||
if stream.write_all(msg).await.is_ok() && stream.flush().await.is_ok() {
|
||||
Some(stream)
|
||||
} else {
|
||||
tracing::warn!(peer = %addr, "failed to write to new peer connection");
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// Send a blocked IP notification to a single peer (one-off, no pool).
|
||||
pub async fn send_blocked_ip(
|
||||
addr: SocketAddr,
|
||||
payload: &BlockedIpPayload,
|
||||
key: &hmac::Key,
|
||||
) -> Result<()> {
|
||||
let json = serde_json::to_vec(payload).context("serialize BlockedIpPayload")?;
|
||||
let msg = proto::encode_message(MessageType::BlockedIp, &json, key);
|
||||
|
||||
let mut stream = tokio::time::timeout(CONNECT_TIMEOUT, TcpStream::connect(addr))
|
||||
.await
|
||||
.context("peer connect timeout")?
|
||||
.context("peer connect failed")?;
|
||||
|
||||
stream.write_all(&msg).await.context("peer write failed")?;
|
||||
stream.flush().await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Listen for incoming peer connections and process messages.
|
||||
pub async fn listen_for_peers(
|
||||
bind_addr: SocketAddr,
|
||||
manager: std::sync::Arc<tokio::sync::Mutex<PeerManager>>,
|
||||
) -> Result<()> {
|
||||
let listener = TcpListener::bind(bind_addr)
|
||||
.await
|
||||
.context("failed to bind peer listener")?;
|
||||
|
||||
tracing::info!(addr = %bind_addr, "peer listener started");
|
||||
|
||||
loop {
|
||||
let (mut stream, peer_addr) = listener.accept().await?;
|
||||
let mgr = manager.clone();
|
||||
|
||||
tokio::spawn(async move {
|
||||
if let Err(e) = handle_peer_connection(&mut stream, peer_addr, &mgr).await {
|
||||
tracing::warn!(peer = %peer_addr, "peer connection ended: {}", e);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/// Handle an incoming peer connection — loops reading messages until the
|
||||
/// remote side disconnects or an I/O error occurs. Sends responses to HELLO
|
||||
/// and Heartbeat messages so the controller can track liveness.
|
||||
///
|
||||
/// Non-fatal errors (bad JSON, unknown payloads) are logged and the loop
|
||||
/// continues. Only I/O errors (connection reset, EOF) break the loop.
|
||||
async fn handle_peer_connection(
|
||||
stream: &mut TcpStream,
|
||||
peer_addr: SocketAddr,
|
||||
manager: &std::sync::Arc<tokio::sync::Mutex<PeerManager>>,
|
||||
) -> Result<()> {
|
||||
// Disable Nagle's algorithm for low-latency responses
|
||||
let _ = stream.set_nodelay(true);
|
||||
|
||||
tracing::info!(peer = %peer_addr, "peer connected, entering read loop");
|
||||
|
||||
loop {
|
||||
// --- Read header: magic(4) + type(1) + payload_len(4) + hmac(32) ---
|
||||
let mut header_buf = [0u8; HEADER_SIZE];
|
||||
stream.read_exact(&mut header_buf).await?;
|
||||
|
||||
let (msg_type, payload_len) = match proto::decode_header(&header_buf) {
|
||||
Some(h) => h,
|
||||
None => {
|
||||
tracing::warn!(
|
||||
peer = %peer_addr,
|
||||
"invalid header (bad magic or unknown type), dropping connection"
|
||||
);
|
||||
anyhow::bail!("invalid header from {}", peer_addr);
|
||||
}
|
||||
};
|
||||
|
||||
if payload_len > MAX_PAYLOAD_SIZE {
|
||||
tracing::warn!(
|
||||
peer = %peer_addr, len = payload_len,
|
||||
"payload too large, dropping connection"
|
||||
);
|
||||
anyhow::bail!("payload too large: {}", payload_len);
|
||||
}
|
||||
|
||||
// --- Read payload (I/O error is fatal) ---
|
||||
let mut payload = vec![0u8; payload_len];
|
||||
if payload_len > 0 {
|
||||
stream.read_exact(&mut payload).await?;
|
||||
}
|
||||
|
||||
// --- Verify HMAC-SHA256 before processing ---
|
||||
{
|
||||
let mgr = manager.lock().await;
|
||||
if !proto::verify_hmac(&header_buf, &payload, mgr.hmac_key()) {
|
||||
tracing::warn!(
|
||||
peer = %peer_addr, msg_type = ?msg_type,
|
||||
"HMAC verification failed — rejecting message"
|
||||
);
|
||||
anyhow::bail!("HMAC verification failed from {}", peer_addr);
|
||||
}
|
||||
}
|
||||
|
||||
// --- Process message (parse errors are non-fatal → continue) ---
|
||||
match msg_type {
|
||||
MessageType::Hello => {
|
||||
let hello: HelloPayload = match serde_json::from_slice(&payload) {
|
||||
Ok(h) => h,
|
||||
Err(e) => {
|
||||
tracing::warn!(peer = %peer_addr, error = %e, "bad Hello JSON, skipping");
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let resp_msg = {
|
||||
let mut mgr = manager.lock().await;
|
||||
// Auto-register the peer so future heartbeats update last_seen
|
||||
mgr.add_peer(peer_addr);
|
||||
mgr.handle_hello(peer_addr, &hello);
|
||||
let resp = mgr.make_hello(mgr.shared_block_count() as u32);
|
||||
let resp_bytes = serde_json::to_vec(&resp)
|
||||
.context("serialize Hello response")?;
|
||||
proto::encode_message(MessageType::Hello, &resp_bytes, mgr.hmac_key())
|
||||
}; // mgr dropped here before I/O
|
||||
stream.write_all(&resp_msg).await?;
|
||||
stream.flush().await?;
|
||||
}
|
||||
MessageType::BlockedIp => {
|
||||
let blocked: BlockedIpPayload = match serde_json::from_slice(&payload) {
|
||||
Ok(b) => b,
|
||||
Err(e) => {
|
||||
tracing::warn!(peer = %peer_addr, error = %e, "bad BlockedIp JSON, skipping");
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let mut mgr = manager.lock().await;
|
||||
mgr.receive_blocked_ip(peer_addr, &blocked);
|
||||
}
|
||||
MessageType::Heartbeat => {
|
||||
let resp_msg = {
|
||||
let mut mgr = manager.lock().await;
|
||||
if let Some(peer) = mgr.peers.get_mut(&peer_addr) {
|
||||
peer.last_seen = Instant::now();
|
||||
}
|
||||
let resp = mgr.make_hello(mgr.shared_block_count() as u32);
|
||||
let resp_bytes = serde_json::to_vec(&resp)
|
||||
.context("serialize Heartbeat response")?;
|
||||
proto::encode_message(MessageType::Hello, &resp_bytes, mgr.hmac_key())
|
||||
}; // mgr dropped here before I/O
|
||||
stream.write_all(&resp_msg).await?;
|
||||
stream.flush().await?;
|
||||
}
|
||||
_ => {
|
||||
tracing::debug!(
|
||||
peer = %peer_addr, msg_type = ?msg_type,
|
||||
"unhandled message type, continuing"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn test_psk() -> &'static [u8] {
|
||||
b"test-secret-key-for-blackwall"
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn peer_manager_add_and_count() {
|
||||
let mut mgr = PeerManager::new("test-node".into(), test_psk());
|
||||
assert_eq!(mgr.peer_count(), 0);
|
||||
|
||||
mgr.add_peer("10.0.0.1:9471".parse().unwrap());
|
||||
assert_eq!(mgr.peer_count(), 1);
|
||||
|
||||
// Duplicate
|
||||
mgr.add_peer("10.0.0.1:9471".parse().unwrap());
|
||||
assert_eq!(mgr.peer_count(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn receive_blocked_ip_with_confidence() {
|
||||
let mut mgr = PeerManager::new("test-node".into(), test_psk());
|
||||
let peer: SocketAddr = "10.0.0.2:9471".parse().unwrap();
|
||||
mgr.add_peer(peer);
|
||||
|
||||
// High confidence — accepted
|
||||
let high = BlockedIpPayload {
|
||||
ip: Ipv4Addr::new(192, 168, 1, 100),
|
||||
reason: "port scan".into(),
|
||||
duration_secs: 600,
|
||||
confidence: 85,
|
||||
};
|
||||
assert!(mgr.receive_blocked_ip(peer, &high).is_some());
|
||||
|
||||
// Low confidence — rejected
|
||||
let low = BlockedIpPayload {
|
||||
ip: Ipv4Addr::new(192, 168, 1, 200),
|
||||
reason: "maybe scan".into(),
|
||||
duration_secs: 60,
|
||||
confidence: 30,
|
||||
};
|
||||
assert!(mgr.receive_blocked_ip(peer, &low).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn make_hello() {
|
||||
let mgr = PeerManager::new("node-42".into(), test_psk());
|
||||
let hello = mgr.make_hello(100);
|
||||
assert_eq!(hello.node_id, "node-42");
|
||||
assert_eq!(hello.blocked_count, 100);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prune_stale_peers() {
|
||||
let mut mgr = PeerManager::new("test".into(), test_psk());
|
||||
mgr.add_peer("10.0.0.1:9471".parse().unwrap());
|
||||
mgr.add_peer("10.0.0.2:9471".parse().unwrap());
|
||||
assert_eq!(mgr.peer_count(), 2);
|
||||
|
||||
// Stale after 0 seconds = prune all
|
||||
mgr.prune_stale_peers(Duration::from_secs(0));
|
||||
assert_eq!(mgr.peer_count(), 0);
|
||||
}
|
||||
}
|
||||
246
blackwall/src/distributed/proto.rs
Executable file
246
blackwall/src/distributed/proto.rs
Executable file
|
|
@ -0,0 +1,246 @@
|
|||
//! Wire protocol for Blackwall peer-to-peer threat intelligence exchange.
|
||||
//!
|
||||
//! Binary protocol with HMAC-SHA256 authentication:
|
||||
//! - Header: magic(4) + type(1) + payload_len(4) + hmac(32)
|
||||
//! - Payload: type-specific JSON data
|
||||
#![allow(dead_code)]
|
||||
//!
|
||||
//! The HMAC covers: magic + type + payload_len + payload (everything except
|
||||
//! the HMAC field itself). A pre-shared key (PSK) must be configured on all
|
||||
//! peers; connections without valid HMAC are rejected immediately.
|
||||
|
||||
use ring::hmac;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::net::Ipv4Addr;
|
||||
|
||||
/// Protocol magic bytes: "BWL\x01"
|
||||
pub const PROTOCOL_MAGIC: [u8; 4] = [0x42, 0x57, 0x4C, 0x01];
|
||||
|
||||
/// Size of the HMAC-SHA256 tag appended to the header.
|
||||
pub const HMAC_SIZE: usize = 32;
|
||||
|
||||
/// Total header size: magic(4) + type(1) + payload_len(4) + hmac(32) = 41.
|
||||
pub const HEADER_SIZE: usize = 4 + 1 + 4 + HMAC_SIZE;
|
||||
|
||||
/// Message types exchanged between peers.
|
||||
#[repr(u8)]
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum MessageType {
|
||||
/// Announce presence to peers
|
||||
Hello = 0x01,
|
||||
/// Share a blocked IP
|
||||
BlockedIp = 0x02,
|
||||
/// Share a JA4 fingerprint observation
|
||||
Ja4Observation = 0x03,
|
||||
/// Heartbeat / keepalive
|
||||
Heartbeat = 0x04,
|
||||
/// Request current threat list
|
||||
SyncRequest = 0x05,
|
||||
/// Response with threat entries
|
||||
SyncResponse = 0x06,
|
||||
}
|
||||
|
||||
impl MessageType {
|
||||
/// Convert from u8 to MessageType.
|
||||
pub fn from_u8(v: u8) -> Option<Self> {
|
||||
match v {
|
||||
0x01 => Some(Self::Hello),
|
||||
0x02 => Some(Self::BlockedIp),
|
||||
0x03 => Some(Self::Ja4Observation),
|
||||
0x04 => Some(Self::Heartbeat),
|
||||
0x05 => Some(Self::SyncRequest),
|
||||
0x06 => Some(Self::SyncResponse),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Hello message payload — node introduces itself.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct HelloPayload {
|
||||
/// Node identifier (hostname or UUID)
|
||||
pub node_id: String,
|
||||
/// Node version
|
||||
pub version: String,
|
||||
/// Number of currently blocked IPs
|
||||
pub blocked_count: u32,
|
||||
}
|
||||
|
||||
/// Blocked IP notification payload.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct BlockedIpPayload {
|
||||
/// The blocked IP address
|
||||
pub ip: Ipv4Addr,
|
||||
/// Reason for blocking
|
||||
pub reason: String,
|
||||
/// Block duration in seconds (0 = permanent)
|
||||
pub duration_secs: u32,
|
||||
/// Confidence score (0-100)
|
||||
pub confidence: u8,
|
||||
}
|
||||
|
||||
/// JA4 fingerprint observation payload.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct Ja4Payload {
|
||||
/// Source IP that sent the TLS ClientHello
|
||||
pub src_ip: Ipv4Addr,
|
||||
/// JA4 fingerprint string
|
||||
pub fingerprint: String,
|
||||
/// Classification: "malicious", "benign", "unknown"
|
||||
pub classification: String,
|
||||
}
|
||||
|
||||
/// Encode a message to bytes with HMAC-SHA256 authentication.
|
||||
///
|
||||
/// Wire format: magic(4) + type(1) + payload_len(4) + hmac(32) + payload
|
||||
/// HMAC covers: magic + type + payload_len + payload.
|
||||
pub fn encode_message(msg_type: MessageType, payload: &[u8], key: &hmac::Key) -> Vec<u8> {
|
||||
let len = payload.len() as u32;
|
||||
let mut buf = Vec::with_capacity(HEADER_SIZE + payload.len());
|
||||
buf.extend_from_slice(&PROTOCOL_MAGIC);
|
||||
buf.push(msg_type as u8);
|
||||
buf.extend_from_slice(&len.to_le_bytes());
|
||||
// Compute HMAC over header fields + payload
|
||||
let mut signing_ctx = hmac::Context::with_key(key);
|
||||
signing_ctx.update(&PROTOCOL_MAGIC);
|
||||
signing_ctx.update(&[msg_type as u8]);
|
||||
signing_ctx.update(&len.to_le_bytes());
|
||||
signing_ctx.update(payload);
|
||||
let tag = signing_ctx.sign();
|
||||
buf.extend_from_slice(tag.as_ref());
|
||||
buf.extend_from_slice(payload);
|
||||
buf
|
||||
}
|
||||
|
||||
/// Decode a message header from bytes. Returns (type, payload_length) if valid.
|
||||
///
|
||||
/// IMPORTANT: This only validates the header structure and magic bytes.
|
||||
/// Call `verify_hmac()` after reading the full payload to authenticate.
|
||||
pub fn decode_header(data: &[u8]) -> Option<(MessageType, usize)> {
|
||||
if data.len() < HEADER_SIZE {
|
||||
return None;
|
||||
}
|
||||
if data[..4] != PROTOCOL_MAGIC {
|
||||
return None;
|
||||
}
|
||||
let msg_type = MessageType::from_u8(data[4])?;
|
||||
let payload_len = u32::from_le_bytes([data[5], data[6], data[7], data[8]]) as usize;
|
||||
Some((msg_type, payload_len))
|
||||
}
|
||||
|
||||
/// Extract the HMAC tag from a decoded header.
|
||||
pub fn extract_hmac(header: &[u8]) -> Option<&[u8]> {
|
||||
if header.len() < HEADER_SIZE {
|
||||
return None;
|
||||
}
|
||||
Some(&header[9..9 + HMAC_SIZE])
|
||||
}
|
||||
|
||||
/// Verify HMAC-SHA256 over header fields + payload.
|
||||
///
|
||||
/// Returns true if the HMAC is valid, false otherwise.
|
||||
pub fn verify_hmac(header: &[u8], payload: &[u8], key: &hmac::Key) -> bool {
|
||||
if header.len() < HEADER_SIZE {
|
||||
return false;
|
||||
}
|
||||
let tag = &header[9..9 + HMAC_SIZE];
|
||||
// Reconstruct signed data: magic + type + payload_len + payload
|
||||
let mut verify_data = Vec::with_capacity(9 + payload.len());
|
||||
verify_data.extend_from_slice(&header[..9]); // magic + type + payload_len
|
||||
verify_data.extend_from_slice(payload);
|
||||
hmac::verify(key, &verify_data, tag).is_ok()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn test_key() -> hmac::Key {
|
||||
hmac::Key::new(hmac::HMAC_SHA256, b"test-secret-key-for-blackwall")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn roundtrip_message() {
|
||||
let key = test_key();
|
||||
let payload = b"test data";
|
||||
let encoded = encode_message(MessageType::Heartbeat, payload, &key);
|
||||
let (msg_type, len) = decode_header(&encoded).expect("decode header");
|
||||
assert_eq!(msg_type, MessageType::Heartbeat);
|
||||
assert_eq!(len, payload.len());
|
||||
assert_eq!(&encoded[HEADER_SIZE..], payload);
|
||||
assert!(verify_hmac(&encoded[..HEADER_SIZE], payload, &key));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_magic_rejected() {
|
||||
let key = test_key();
|
||||
let mut data = encode_message(MessageType::Hello, b"hi", &key);
|
||||
data[0] = 0xFF; // Corrupt magic
|
||||
assert!(decode_header(&data).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn too_short_rejected() {
|
||||
assert!(decode_header(&[0; 5]).is_none());
|
||||
assert!(decode_header(&[0; 40]).is_none()); // < HEADER_SIZE (41)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hmac_tamper_detected() {
|
||||
let key = test_key();
|
||||
let mut encoded = encode_message(MessageType::BlockedIp, b"payload", &key);
|
||||
// Tamper with payload
|
||||
if let Some(last) = encoded.last_mut() {
|
||||
*last ^= 0xFF;
|
||||
}
|
||||
let (_, len) = decode_header(&encoded).expect("header still valid");
|
||||
let payload = &encoded[HEADER_SIZE..HEADER_SIZE + len];
|
||||
assert!(!verify_hmac(&encoded[..HEADER_SIZE], payload, &key));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn wrong_key_rejected() {
|
||||
let key1 = test_key();
|
||||
let key2 = hmac::Key::new(hmac::HMAC_SHA256, b"wrong-key");
|
||||
let encoded = encode_message(MessageType::Hello, b"data", &key1);
|
||||
let (_, len) = decode_header(&encoded).expect("decode");
|
||||
let payload = &encoded[HEADER_SIZE..HEADER_SIZE + len];
|
||||
assert!(!verify_hmac(&encoded[..HEADER_SIZE], payload, &key2));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn all_message_types() {
|
||||
for byte in 0x01..=0x06 {
|
||||
assert!(MessageType::from_u8(byte).is_some());
|
||||
}
|
||||
assert!(MessageType::from_u8(0x00).is_none());
|
||||
assert!(MessageType::from_u8(0xFF).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hello_payload_serialization() {
|
||||
let hello = HelloPayload {
|
||||
node_id: "node-1".into(),
|
||||
version: "0.1.0".into(),
|
||||
blocked_count: 42,
|
||||
};
|
||||
let json = serde_json::to_vec(&hello).unwrap();
|
||||
let decoded: HelloPayload = serde_json::from_slice(&json).unwrap();
|
||||
assert_eq!(decoded.node_id, "node-1");
|
||||
assert_eq!(decoded.blocked_count, 42);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn blocked_ip_payload_serialization() {
|
||||
let blocked = BlockedIpPayload {
|
||||
ip: Ipv4Addr::new(192, 168, 1, 100),
|
||||
reason: "port scan".into(),
|
||||
duration_secs: 600,
|
||||
confidence: 85,
|
||||
};
|
||||
let json = serde_json::to_vec(&blocked).unwrap();
|
||||
let decoded: BlockedIpPayload = serde_json::from_slice(&json).unwrap();
|
||||
assert_eq!(decoded.ip, Ipv4Addr::new(192, 168, 1, 100));
|
||||
assert_eq!(decoded.confidence, 85);
|
||||
}
|
||||
}
|
||||
214
blackwall/src/dpi/dns.rs
Executable file
214
blackwall/src/dpi/dns.rs
Executable file
|
|
@ -0,0 +1,214 @@
|
|||
//! DNS query/response dissector.
|
||||
//!
|
||||
//! Parses DNS wire format from raw bytes.
|
||||
//! Extracts query name, type, and detects tunneling indicators.
|
||||
|
||||
/// Extracted DNS query metadata.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct DnsInfo {
|
||||
/// Transaction ID
|
||||
pub id: u16,
|
||||
/// Whether this is a query (false) or response (true)
|
||||
pub is_response: bool,
|
||||
/// Number of questions
|
||||
pub question_count: u16,
|
||||
/// First query domain name (if present)
|
||||
pub query_name: Option<String>,
|
||||
/// First query type (A=1, AAAA=28, MX=15, TXT=16, CNAME=5)
|
||||
pub query_type: Option<u16>,
|
||||
}
|
||||
|
||||
/// Maximum DNS name length to parse (prevents excessive processing).
|
||||
const MAX_DNS_NAME_LEN: usize = 253;
|
||||
/// Maximum label count to prevent infinite loops on malformed packets.
|
||||
const MAX_LABELS: usize = 128;
|
||||
|
||||
/// Parse a DNS query/response from raw bytes (UDP payload).
|
||||
pub fn parse_query(data: &[u8]) -> Option<DnsInfo> {
|
||||
// DNS header is 12 bytes minimum
|
||||
if data.len() < 12 {
|
||||
return None;
|
||||
}
|
||||
|
||||
let id = u16::from_be_bytes([data[0], data[1]]);
|
||||
let flags = u16::from_be_bytes([data[2], data[3]]);
|
||||
let is_response = (flags & 0x8000) != 0;
|
||||
let question_count = u16::from_be_bytes([data[4], data[5]]);
|
||||
|
||||
// Sanity: must have at least 1 question
|
||||
if question_count == 0 || question_count > 256 {
|
||||
return None;
|
||||
}
|
||||
|
||||
// Parse first question name starting at offset 12
|
||||
let (query_name, offset) = parse_dns_name(data, 12)?;
|
||||
|
||||
// Parse query type (2 bytes after name)
|
||||
let query_type = if offset + 2 <= data.len() {
|
||||
Some(u16::from_be_bytes([data[offset], data[offset + 1]]))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
Some(DnsInfo {
|
||||
id,
|
||||
is_response,
|
||||
question_count,
|
||||
query_name: Some(query_name),
|
||||
query_type,
|
||||
})
|
||||
}
|
||||
|
||||
/// Parse a DNS domain name from wire format. Returns (name, bytes_consumed_offset).
|
||||
fn parse_dns_name(data: &[u8], start: usize) -> Option<(String, usize)> {
|
||||
let mut name = String::new();
|
||||
let mut pos = start;
|
||||
let mut labels = 0;
|
||||
|
||||
loop {
|
||||
if pos >= data.len() || labels >= MAX_LABELS {
|
||||
return None;
|
||||
}
|
||||
|
||||
let label_len = data[pos] as usize;
|
||||
if label_len == 0 {
|
||||
pos += 1;
|
||||
break;
|
||||
}
|
||||
|
||||
// Compression pointer (0xC0 prefix) — not following for simplicity
|
||||
if label_len & 0xC0 == 0xC0 {
|
||||
pos += 2;
|
||||
break;
|
||||
}
|
||||
|
||||
if label_len > 63 {
|
||||
return None; // Invalid label length
|
||||
}
|
||||
|
||||
pos += 1;
|
||||
if pos + label_len > data.len() {
|
||||
return None;
|
||||
}
|
||||
|
||||
if !name.is_empty() {
|
||||
name.push('.');
|
||||
}
|
||||
|
||||
let label = std::str::from_utf8(&data[pos..pos + label_len]).ok()?;
|
||||
name.push_str(label);
|
||||
|
||||
if name.len() > MAX_DNS_NAME_LEN {
|
||||
return None;
|
||||
}
|
||||
|
||||
pos += label_len;
|
||||
labels += 1;
|
||||
}
|
||||
|
||||
if name.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
Some((name, pos))
|
||||
}
|
||||
|
||||
/// Heuristic: check if a DNS query name looks like DNS tunneling.
|
||||
/// High entropy names, very long labels, and many subdomains are suspicious.
|
||||
pub fn is_tunneling_suspect(name: &str) -> bool {
|
||||
// Long overall name
|
||||
if name.len() > 60 {
|
||||
return true;
|
||||
}
|
||||
|
||||
let labels: Vec<&str> = name.split('.').collect();
|
||||
|
||||
// Many subdomain levels
|
||||
if labels.len() > 6 {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Any individual label is unusually long (>30 chars suggests encoded data)
|
||||
for label in &labels {
|
||||
if label.len() > 30 {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
// High ratio of digits/hex chars in labels (encoded payload)
|
||||
let total_chars: usize = labels.iter().take(labels.len().saturating_sub(2)).map(|l| l.len()).sum();
|
||||
if total_chars > 10 {
|
||||
let hex_chars: usize = labels
|
||||
.iter()
|
||||
.take(labels.len().saturating_sub(2))
|
||||
.flat_map(|l| l.chars())
|
||||
.filter(|c| c.is_ascii_hexdigit() && !c.is_ascii_alphabetic())
|
||||
.count();
|
||||
if hex_chars * 3 > total_chars {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
false
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn build_dns_query(name: &str, qtype: u16) -> Vec<u8> {
|
||||
let mut pkt = vec![
|
||||
0x12, 0x34, // Transaction ID
|
||||
0x01, 0x00, // Flags: standard query
|
||||
0x00, 0x01, // Questions: 1
|
||||
0x00, 0x00, // Answers: 0
|
||||
0x00, 0x00, // Authority: 0
|
||||
0x00, 0x00, // Additional: 0
|
||||
];
|
||||
// Encode name
|
||||
for label in name.split('.') {
|
||||
pkt.push(label.len() as u8);
|
||||
pkt.extend_from_slice(label.as_bytes());
|
||||
}
|
||||
pkt.push(0); // Root terminator
|
||||
pkt.extend_from_slice(&qtype.to_be_bytes());
|
||||
pkt.extend_from_slice(&1u16.to_be_bytes()); // Class IN
|
||||
pkt
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_simple_a_query() {
|
||||
let pkt = build_dns_query("example.com", 1);
|
||||
let info = parse_query(&pkt).unwrap();
|
||||
assert_eq!(info.id, 0x1234);
|
||||
assert!(!info.is_response);
|
||||
assert_eq!(info.question_count, 1);
|
||||
assert_eq!(info.query_name, Some("example.com".into()));
|
||||
assert_eq!(info.query_type, Some(1)); // A record
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_txt_query() {
|
||||
let pkt = build_dns_query("tunnel.evil.com", 16);
|
||||
let info = parse_query(&pkt).unwrap();
|
||||
assert_eq!(info.query_name, Some("tunnel.evil.com".into()));
|
||||
assert_eq!(info.query_type, Some(16)); // TXT record
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reject_too_short() {
|
||||
assert!(parse_query(&[0; 6]).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tunneling_detection() {
|
||||
assert!(!is_tunneling_suspect("google.com"));
|
||||
assert!(!is_tunneling_suspect("www.example.com"));
|
||||
assert!(is_tunneling_suspect(
|
||||
"aGVsbG8gd29ybGQgdGhpcyBpcyBlbmNvZGVk.tunnel.evil.com"
|
||||
));
|
||||
assert!(is_tunneling_suspect(
|
||||
"a.b.c.d.e.f.g.evil.com"
|
||||
));
|
||||
}
|
||||
}
|
||||
174
blackwall/src/dpi/http.rs
Executable file
174
blackwall/src/dpi/http.rs
Executable file
|
|
@ -0,0 +1,174 @@
|
|||
//! HTTP request/response dissector.
|
||||
//!
|
||||
//! Parses HTTP/1.x request lines and headers from raw bytes.
|
||||
//! Extracts method, path, Host header, User-Agent, and Content-Type.
|
||||
|
||||
/// Extracted HTTP request metadata.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct HttpInfo {
|
||||
/// HTTP method (GET, POST, PUT, etc.)
|
||||
pub method: String,
|
||||
/// Request path
|
||||
pub path: String,
|
||||
/// HTTP version (e.g., "1.1", "1.0")
|
||||
pub version: String,
|
||||
/// Host header value
|
||||
pub host: Option<String>,
|
||||
/// User-Agent header value
|
||||
pub user_agent: Option<String>,
|
||||
/// Content-Type header value
|
||||
pub content_type: Option<String>,
|
||||
}
|
||||
|
||||
/// Known suspicious paths that scanners frequently probe.
|
||||
const SUSPICIOUS_PATHS: &[&str] = &[
|
||||
"/wp-login.php",
|
||||
"/wp-admin",
|
||||
"/xmlrpc.php",
|
||||
"/phpmyadmin",
|
||||
"/admin",
|
||||
"/administrator",
|
||||
"/.env",
|
||||
"/.git/config",
|
||||
"/config.php",
|
||||
"/shell",
|
||||
"/cmd",
|
||||
"/eval",
|
||||
"/actuator",
|
||||
"/solr",
|
||||
"/console",
|
||||
"/manager/html",
|
||||
"/cgi-bin/",
|
||||
"/../",
|
||||
];
|
||||
|
||||
/// Known malicious User-Agent patterns.
|
||||
const SUSPICIOUS_USER_AGENTS: &[&str] = &[
|
||||
"sqlmap",
|
||||
"nikto",
|
||||
"nmap",
|
||||
"masscan",
|
||||
"zgrab",
|
||||
"gobuster",
|
||||
"dirbuster",
|
||||
"wpscan",
|
||||
"nuclei",
|
||||
"httpx",
|
||||
"curl/",
|
||||
"python-requests",
|
||||
"go-http-client",
|
||||
];
|
||||
|
||||
/// Parse an HTTP request from raw bytes.
|
||||
pub fn parse_request(data: &[u8]) -> Option<HttpInfo> {
|
||||
// HTTP requests start with a method followed by space
|
||||
let text = std::str::from_utf8(data).ok()?;
|
||||
|
||||
// Find the request line (first line)
|
||||
let request_line = text.lines().next()?;
|
||||
let mut parts = request_line.splitn(3, ' ');
|
||||
|
||||
let method = parts.next()?;
|
||||
// Validate method
|
||||
if !matches!(
|
||||
method,
|
||||
"GET" | "POST" | "PUT" | "DELETE" | "HEAD" | "OPTIONS" | "PATCH" | "CONNECT" | "TRACE"
|
||||
) {
|
||||
return None;
|
||||
}
|
||||
|
||||
let path = parts.next().unwrap_or("/");
|
||||
let version_str = parts.next().unwrap_or("HTTP/1.1");
|
||||
let version = version_str.strip_prefix("HTTP/").unwrap_or("1.1");
|
||||
|
||||
// Parse headers
|
||||
let mut host = None;
|
||||
let mut user_agent = None;
|
||||
let mut content_type = None;
|
||||
|
||||
for line in text.lines().skip(1) {
|
||||
if line.is_empty() {
|
||||
break; // End of headers
|
||||
}
|
||||
if let Some((name, value)) = line.split_once(':') {
|
||||
let name_lower = name.trim().to_lowercase();
|
||||
let value = value.trim();
|
||||
match name_lower.as_str() {
|
||||
"host" => host = Some(value.to_string()),
|
||||
"user-agent" => user_agent = Some(value.to_string()),
|
||||
"content-type" => content_type = Some(value.to_string()),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Some(HttpInfo {
|
||||
method: method.to_string(),
|
||||
path: path.to_string(),
|
||||
version: version.to_string(),
|
||||
host,
|
||||
user_agent,
|
||||
content_type,
|
||||
})
|
||||
}
|
||||
|
||||
/// Check if the request path is suspicious (known scanner targets).
|
||||
pub fn is_suspicious_path(path: &str) -> bool {
|
||||
let lower = path.to_lowercase();
|
||||
SUSPICIOUS_PATHS.iter().any(|p| lower.contains(p))
|
||||
}
|
||||
|
||||
/// Check if the User-Agent matches known scanning tools.
|
||||
pub fn is_suspicious_user_agent(ua: &str) -> bool {
|
||||
let lower = ua.to_lowercase();
|
||||
SUSPICIOUS_USER_AGENTS.iter().any(|p| lower.contains(p))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn parse_simple_get() {
|
||||
let data = b"GET / HTTP/1.1\r\nHost: example.com\r\n\r\n";
|
||||
let info = parse_request(data).unwrap();
|
||||
assert_eq!(info.method, "GET");
|
||||
assert_eq!(info.path, "/");
|
||||
assert_eq!(info.version, "1.1");
|
||||
assert_eq!(info.host, Some("example.com".into()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_post_with_headers() {
|
||||
let data = b"POST /api/login HTTP/1.1\r\nHost: app.local\r\n\
|
||||
User-Agent: Mozilla/5.0\r\nContent-Type: application/json\r\n\r\n";
|
||||
let info = parse_request(data).unwrap();
|
||||
assert_eq!(info.method, "POST");
|
||||
assert_eq!(info.path, "/api/login");
|
||||
assert_eq!(info.user_agent, Some("Mozilla/5.0".into()));
|
||||
assert_eq!(info.content_type, Some("application/json".into()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reject_non_http() {
|
||||
assert!(parse_request(b"SSH-2.0-OpenSSH\r\n").is_none());
|
||||
assert!(parse_request(b"\x00\x01\x02").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn suspicious_path_detection() {
|
||||
assert!(is_suspicious_path("/wp-login.php"));
|
||||
assert!(is_suspicious_path("/foo/../etc/passwd"));
|
||||
assert!(is_suspicious_path("/.env"));
|
||||
assert!(!is_suspicious_path("/index.html"));
|
||||
assert!(!is_suspicious_path("/api/v1/users"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn suspicious_user_agent_detection() {
|
||||
assert!(is_suspicious_user_agent("sqlmap/1.7.2"));
|
||||
assert!(is_suspicious_user_agent("Nikto/2.1.6"));
|
||||
assert!(is_suspicious_user_agent("python-requests/2.28.0"));
|
||||
assert!(!is_suspicious_user_agent("Mozilla/5.0 (Windows NT 10.0; Win64; x64)"));
|
||||
}
|
||||
}
|
||||
73
blackwall/src/dpi/mod.rs
Executable file
73
blackwall/src/dpi/mod.rs
Executable file
|
|
@ -0,0 +1,73 @@
|
|||
//! Deep Packet Inspection (DPI) dissectors for protocol-level analysis.
|
||||
//!
|
||||
//! Operates on raw connection bytes captured from network streams.
|
||||
//! Dissectors extract protocol metadata for threat classification.
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub mod dns;
|
||||
#[allow(dead_code)]
|
||||
pub mod http;
|
||||
#[allow(dead_code)]
|
||||
pub mod ssh;
|
||||
|
||||
/// Protocol identified by DPI analysis.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
#[allow(dead_code)]
|
||||
pub enum DetectedProtocol {
|
||||
Http(http::HttpInfo),
|
||||
Dns(dns::DnsInfo),
|
||||
Ssh(ssh::SshInfo),
|
||||
Unknown,
|
||||
}
|
||||
|
||||
/// Attempt to identify the protocol from the first bytes of a connection.
|
||||
#[allow(dead_code)]
|
||||
pub fn identify_protocol(data: &[u8]) -> DetectedProtocol {
|
||||
// Try HTTP first (most common on redirected traffic)
|
||||
if let Some(info) = http::parse_request(data) {
|
||||
return DetectedProtocol::Http(info);
|
||||
}
|
||||
// Try SSH banner
|
||||
if let Some(info) = ssh::parse_banner(data) {
|
||||
return DetectedProtocol::Ssh(info);
|
||||
}
|
||||
// Try DNS (UDP payload)
|
||||
if let Some(info) = dns::parse_query(data) {
|
||||
return DetectedProtocol::Dns(info);
|
||||
}
|
||||
DetectedProtocol::Unknown
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn identify_http_get() {
|
||||
let data = b"GET /admin HTTP/1.1\r\nHost: example.com\r\n\r\n";
|
||||
match identify_protocol(data) {
|
||||
DetectedProtocol::Http(info) => {
|
||||
assert_eq!(info.method, "GET");
|
||||
assert_eq!(info.path, "/admin");
|
||||
}
|
||||
other => panic!("expected Http, got {:?}", other),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn identify_ssh_banner() {
|
||||
let data = b"SSH-2.0-OpenSSH_9.6p1 Ubuntu-3ubuntu13.5\r\n";
|
||||
match identify_protocol(data) {
|
||||
DetectedProtocol::Ssh(info) => {
|
||||
assert!(info.version.contains("OpenSSH"));
|
||||
}
|
||||
other => panic!("expected Ssh, got {:?}", other),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn identify_unknown() {
|
||||
let data = b"\x00\x01\x02\x03random binary";
|
||||
assert_eq!(identify_protocol(data), DetectedProtocol::Unknown);
|
||||
}
|
||||
}
|
||||
103
blackwall/src/dpi/ssh.rs
Executable file
103
blackwall/src/dpi/ssh.rs
Executable file
|
|
@ -0,0 +1,103 @@
|
|||
//! SSH banner/version dissector.
|
||||
//!
|
||||
//! Parses SSH protocol version exchange strings.
|
||||
//! Extracts software version and detects known scanning tools.
|
||||
|
||||
/// Extracted SSH banner metadata.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct SshInfo {
|
||||
/// Full version string (e.g., "SSH-2.0-OpenSSH_9.6p1 Ubuntu-3ubuntu13.5")
|
||||
pub version: String,
|
||||
/// Protocol version ("2.0" or "1.99")
|
||||
pub protocol: String,
|
||||
/// Software identifier (e.g., "OpenSSH_9.6p1")
|
||||
pub software: String,
|
||||
/// Optional comment/OS info
|
||||
pub comment: Option<String>,
|
||||
}
|
||||
|
||||
/// Known SSH scanning/attack tool identifiers.
|
||||
const SUSPICIOUS_SSH_SOFTWARE: &[&str] = &[
|
||||
"libssh", // Frequently used by bots
|
||||
"paramiko", // Python SSH library, common in automated attacks
|
||||
"putty", // PuTTY — sometimes spoofed
|
||||
"go", // Go SSH libraries (automated scanners)
|
||||
"asyncssh", // Python async SSH
|
||||
"nmap", // Nmap SSH scanning
|
||||
"dropbear_2012", // Very old, likely compromised device
|
||||
"dropbear_2014",
|
||||
"sshlibrary",
|
||||
"russh",
|
||||
];
|
||||
|
||||
/// Parse an SSH banner from raw connection bytes.
|
||||
pub fn parse_banner(data: &[u8]) -> Option<SshInfo> {
|
||||
let text = std::str::from_utf8(data).ok()?;
|
||||
let line = text.lines().next()?;
|
||||
|
||||
// SSH banner format: SSH-protoversion-softwareversion [SP comments]
|
||||
if !line.starts_with("SSH-") {
|
||||
return None;
|
||||
}
|
||||
|
||||
let banner = line.strip_prefix("SSH-")?;
|
||||
|
||||
// Split into protocol-softwareversion and optional comment
|
||||
let (proto_sw, comment) = match banner.split_once(' ') {
|
||||
Some((ps, c)) => (ps, Some(c.trim().to_string())),
|
||||
None => (banner.trim_end_matches('\r'), None),
|
||||
};
|
||||
|
||||
// Split protocol and software
|
||||
let (protocol, software) = proto_sw.split_once('-')?;
|
||||
|
||||
Some(SshInfo {
|
||||
version: line.to_string(),
|
||||
protocol: protocol.to_string(),
|
||||
software: software.to_string(),
|
||||
comment,
|
||||
})
|
||||
}
|
||||
|
||||
/// Check if the SSH software banner matches known scanning/attack tools.
|
||||
pub fn is_suspicious_software(software: &str) -> bool {
|
||||
let lower = software.to_lowercase();
|
||||
SUSPICIOUS_SSH_SOFTWARE.iter().any(|s| lower.contains(s))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn parse_openssh_banner() {
|
||||
let data = b"SSH-2.0-OpenSSH_9.6p1 Ubuntu-3ubuntu13.5\r\n";
|
||||
let info = parse_banner(data).unwrap();
|
||||
assert_eq!(info.protocol, "2.0");
|
||||
assert_eq!(info.software, "OpenSSH_9.6p1");
|
||||
assert_eq!(info.comment, Some("Ubuntu-3ubuntu13.5".into()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_no_comment() {
|
||||
let data = b"SSH-2.0-dropbear_2022.82\r\n";
|
||||
let info = parse_banner(data).unwrap();
|
||||
assert_eq!(info.protocol, "2.0");
|
||||
assert_eq!(info.software, "dropbear_2022.82");
|
||||
assert!(info.comment.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reject_non_ssh() {
|
||||
assert!(parse_banner(b"HTTP/1.1 200 OK\r\n").is_none());
|
||||
assert!(parse_banner(b"\x00\x01\x02").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn suspicious_software_detection() {
|
||||
assert!(is_suspicious_software("libssh-0.9.6"));
|
||||
assert!(is_suspicious_software("Paramiko_3.4.0"));
|
||||
assert!(!is_suspicious_software("OpenSSH_9.6p1"));
|
||||
assert!(!is_suspicious_software("dropbear_2022.82"));
|
||||
}
|
||||
}
|
||||
113
blackwall/src/events.rs
Executable file
113
blackwall/src/events.rs
Executable file
|
|
@ -0,0 +1,113 @@
|
|||
use anyhow::Result;
|
||||
use aya::maps::{MapData, RingBuf};
|
||||
use common::{DpiEvent, EgressEvent, PacketEvent, TlsComponentsEvent};
|
||||
use crossbeam_queue::SegQueue;
|
||||
use std::os::fd::AsRawFd;
|
||||
use std::sync::Arc;
|
||||
use tokio::io::unix::AsyncFd;
|
||||
|
||||
/// Asynchronously consume PacketEvents from the eBPF RingBuf and push them
|
||||
/// into a lock-free queue for downstream processing.
|
||||
pub async fn consume_events(
|
||||
ring_buf: RingBuf<MapData>,
|
||||
event_tx: Arc<SegQueue<PacketEvent>>,
|
||||
) -> Result<()> {
|
||||
let mut ring_buf = ring_buf;
|
||||
let async_fd = AsyncFd::new(ring_buf.as_raw_fd())?;
|
||||
|
||||
loop {
|
||||
// Wait for readability (epoll-based, no busy-spin)
|
||||
let mut guard = async_fd.readable().await?;
|
||||
|
||||
// Drain all available events
|
||||
while let Some(event_data) = ring_buf.next() {
|
||||
if event_data.len() < core::mem::size_of::<PacketEvent>() {
|
||||
continue;
|
||||
}
|
||||
// SAFETY: PacketEvent is #[repr(C)] with known layout, Pod-safe.
|
||||
// eBPF wrote exactly sizeof(PacketEvent) bytes via reserve/submit.
|
||||
let event: &PacketEvent = unsafe { &*(event_data.as_ptr() as *const PacketEvent) };
|
||||
event_tx.push(*event);
|
||||
}
|
||||
|
||||
guard.clear_ready();
|
||||
}
|
||||
}
|
||||
|
||||
/// Asynchronously consume TlsComponentsEvents from the eBPF TLS_EVENTS RingBuf
|
||||
/// and push them into a lock-free queue for JA4 fingerprint assembly.
|
||||
pub async fn consume_tls_events(
|
||||
ring_buf: RingBuf<MapData>,
|
||||
tls_tx: Arc<SegQueue<TlsComponentsEvent>>,
|
||||
) -> Result<()> {
|
||||
let mut ring_buf = ring_buf;
|
||||
let async_fd = AsyncFd::new(ring_buf.as_raw_fd())?;
|
||||
|
||||
loop {
|
||||
let mut guard = async_fd.readable().await?;
|
||||
|
||||
while let Some(event_data) = ring_buf.next() {
|
||||
if event_data.len() < core::mem::size_of::<TlsComponentsEvent>() {
|
||||
continue;
|
||||
}
|
||||
// SAFETY: TlsComponentsEvent is #[repr(C)] Pod-safe, written by eBPF reserve/submit.
|
||||
let event: &TlsComponentsEvent =
|
||||
unsafe { &*(event_data.as_ptr() as *const TlsComponentsEvent) };
|
||||
tls_tx.push(*event);
|
||||
}
|
||||
|
||||
guard.clear_ready();
|
||||
}
|
||||
}
|
||||
|
||||
/// Asynchronously consume EgressEvents from the eBPF EGRESS_EVENTS RingBuf
|
||||
/// and push them into a lock-free queue for outbound traffic analysis.
|
||||
pub async fn consume_egress_events(
|
||||
ring_buf: RingBuf<MapData>,
|
||||
egress_tx: Arc<SegQueue<EgressEvent>>,
|
||||
) -> Result<()> {
|
||||
let mut ring_buf = ring_buf;
|
||||
let async_fd = AsyncFd::new(ring_buf.as_raw_fd())?;
|
||||
|
||||
loop {
|
||||
let mut guard = async_fd.readable().await?;
|
||||
|
||||
while let Some(event_data) = ring_buf.next() {
|
||||
if event_data.len() < core::mem::size_of::<EgressEvent>() {
|
||||
continue;
|
||||
}
|
||||
// SAFETY: EgressEvent is #[repr(C)] Pod-safe, written by eBPF reserve/submit.
|
||||
let event: &EgressEvent =
|
||||
unsafe { &*(event_data.as_ptr() as *const EgressEvent) };
|
||||
egress_tx.push(*event);
|
||||
}
|
||||
|
||||
guard.clear_ready();
|
||||
}
|
||||
}
|
||||
|
||||
/// Asynchronously consume DpiEvents from the eBPF DPI_EVENTS RingBuf
|
||||
/// and push them into a lock-free queue for protocol-level analysis.
|
||||
pub async fn consume_dpi_events(
|
||||
ring_buf: RingBuf<MapData>,
|
||||
dpi_tx: Arc<SegQueue<DpiEvent>>,
|
||||
) -> Result<()> {
|
||||
let mut ring_buf = ring_buf;
|
||||
let async_fd = AsyncFd::new(ring_buf.as_raw_fd())?;
|
||||
|
||||
loop {
|
||||
let mut guard = async_fd.readable().await?;
|
||||
|
||||
while let Some(event_data) = ring_buf.next() {
|
||||
if event_data.len() < core::mem::size_of::<DpiEvent>() {
|
||||
continue;
|
||||
}
|
||||
// SAFETY: DpiEvent is #[repr(C)] Pod-safe, written by eBPF reserve/submit.
|
||||
let event: &DpiEvent =
|
||||
unsafe { &*(event_data.as_ptr() as *const DpiEvent) };
|
||||
dpi_tx.push(*event);
|
||||
}
|
||||
|
||||
guard.clear_ready();
|
||||
}
|
||||
}
|
||||
230
blackwall/src/feeds.rs
Executable file
230
blackwall/src/feeds.rs
Executable file
|
|
@ -0,0 +1,230 @@
|
|||
//! Threat feed fetcher: downloads IP blocklists and updates eBPF maps.
|
||||
//!
|
||||
//! Supports plain-text feeds (one IP per line, # comments).
|
||||
//! Popular sources: Firehol level1, abuse.ch feodo, Spamhaus DROP.
|
||||
//! Handles both single IPs and CIDR ranges (e.g., `10.0.0.0/8`).
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use http_body_util::{BodyExt, Empty, Limited};
|
||||
use hyper::body::Bytes;
|
||||
use hyper::Request;
|
||||
use hyper_util::client::legacy::Client;
|
||||
use hyper_util::rt::TokioExecutor;
|
||||
use std::net::Ipv4Addr;
|
||||
use std::time::Duration;
|
||||
|
||||
/// Maximum IPs to ingest from a single feed (prevents memory exhaustion).
|
||||
const MAX_IPS_PER_FEED: usize = 50_000;
|
||||
/// HTTP request timeout per feed.
|
||||
const FEED_TIMEOUT_SECS: u64 = 30;
|
||||
/// Maximum response body size (10 MB) — prevents memory exhaustion from rogue feeds.
|
||||
const MAX_BODY_BYTES: usize = 10 * 1024 * 1024;
|
||||
|
||||
/// A single entry parsed from a threat feed.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum FeedEntry {
|
||||
/// Single IP address.
|
||||
Single(Ipv4Addr),
|
||||
/// CIDR range (network address + prefix length).
|
||||
Cidr(Ipv4Addr, u8),
|
||||
}
|
||||
|
||||
/// A configured threat feed source.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct FeedSource {
|
||||
/// Human-readable name for logging.
|
||||
pub name: String,
|
||||
/// URL to fetch (must return text/plain with one IP per line).
|
||||
pub url: String,
|
||||
/// Block duration in seconds (0 = permanent until next refresh).
|
||||
pub block_duration_secs: u32,
|
||||
}
|
||||
|
||||
/// Fetch a single feed and return parsed entries (single IPs or CIDR ranges).
|
||||
pub async fn fetch_feed(source: &FeedSource) -> Result<Vec<FeedEntry>> {
|
||||
let https = hyper_rustls::HttpsConnectorBuilder::new()
|
||||
.with_webpki_roots()
|
||||
.https_or_http()
|
||||
.enable_http1()
|
||||
.build();
|
||||
let client = Client::builder(TokioExecutor::new()).build(https);
|
||||
let req = Request::get(&source.url)
|
||||
.header("User-Agent", "Blackwall/0.1")
|
||||
.body(Empty::<Bytes>::new())
|
||||
.context("invalid feed URL")?;
|
||||
|
||||
let resp = tokio::time::timeout(
|
||||
Duration::from_secs(FEED_TIMEOUT_SECS),
|
||||
client.request(req),
|
||||
)
|
||||
.await
|
||||
.context("feed request timed out")?
|
||||
.context("feed HTTP request failed")?;
|
||||
|
||||
if !resp.status().is_success() {
|
||||
anyhow::bail!(
|
||||
"feed {} returned HTTP {}",
|
||||
source.name,
|
||||
resp.status()
|
||||
);
|
||||
}
|
||||
|
||||
let body_bytes = Limited::new(resp.into_body(), MAX_BODY_BYTES)
|
||||
.collect()
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("failed to read feed body (possibly exceeded 10MB limit): {}", e))?
|
||||
.to_bytes();
|
||||
|
||||
let body = String::from_utf8_lossy(&body_bytes);
|
||||
let entries = parse_feed_body(&body);
|
||||
|
||||
if entries.len() >= MAX_IPS_PER_FEED {
|
||||
tracing::warn!(
|
||||
feed = %source.name,
|
||||
max = MAX_IPS_PER_FEED,
|
||||
"feed truncated at max entries"
|
||||
);
|
||||
}
|
||||
|
||||
Ok(entries)
|
||||
}
|
||||
|
||||
/// Parse feed body text into entries (reusable for testing).
|
||||
fn parse_feed_body(body: &str) -> Vec<FeedEntry> {
|
||||
let mut entries = Vec::new();
|
||||
|
||||
for line in body.lines() {
|
||||
let trimmed = line.trim();
|
||||
// Skip comments and empty lines
|
||||
if trimmed.is_empty() || trimmed.starts_with('#') || trimmed.starts_with(';') {
|
||||
continue;
|
||||
}
|
||||
// Some feeds have "IP<tab>info" or "IP # comment" format
|
||||
let ip_str = trimmed.split_whitespace().next().unwrap_or("");
|
||||
|
||||
if let Some(entry) = parse_ip_or_cidr(ip_str) {
|
||||
entries.push(entry);
|
||||
if entries.len() >= MAX_IPS_PER_FEED {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
entries
|
||||
}
|
||||
|
||||
/// Parse a single token as either `IP/prefix` (CIDR) or plain IP.
|
||||
fn parse_ip_or_cidr(s: &str) -> Option<FeedEntry> {
|
||||
if let Some((ip_part, prefix_part)) = s.split_once('/') {
|
||||
let ip: Ipv4Addr = ip_part.parse().ok()?;
|
||||
let prefix: u8 = prefix_part.parse().ok()?;
|
||||
if prefix > 32 {
|
||||
return None;
|
||||
}
|
||||
Some(FeedEntry::Cidr(ip, prefix))
|
||||
} else {
|
||||
let ip: Ipv4Addr = s.parse().ok()?;
|
||||
Some(FeedEntry::Single(ip))
|
||||
}
|
||||
}
|
||||
|
||||
/// Fetch all configured feeds and return combined unique entries with block durations.
|
||||
pub async fn fetch_all_feeds(sources: &[FeedSource]) -> Vec<(FeedEntry, u32)> {
|
||||
let mut all_entries: Vec<(FeedEntry, u32)> = Vec::new();
|
||||
let mut seen_ips = std::collections::HashSet::new();
|
||||
let mut seen_cidrs = std::collections::HashSet::new();
|
||||
|
||||
for source in sources {
|
||||
match fetch_feed(source).await {
|
||||
Ok(entries) => {
|
||||
let count = entries.len();
|
||||
for entry in entries {
|
||||
let is_new = match &entry {
|
||||
FeedEntry::Single(ip) => seen_ips.insert(*ip),
|
||||
FeedEntry::Cidr(ip, prefix) => seen_cidrs.insert((*ip, *prefix)),
|
||||
};
|
||||
if is_new {
|
||||
all_entries.push((entry, source.block_duration_secs));
|
||||
}
|
||||
}
|
||||
tracing::info!(
|
||||
feed = %source.name,
|
||||
new_entries = count,
|
||||
total = all_entries.len(),
|
||||
"feed fetched successfully"
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
feed = %source.name,
|
||||
error = %e,
|
||||
"feed fetch failed — skipping"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
all_entries
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn parse_plain_ip_list() {
|
||||
let body = "# Comment line\n\
|
||||
192.168.1.1\n\
|
||||
10.0.0.1\n\
|
||||
\n\
|
||||
; Another comment\n\
|
||||
172.16.0.1\t# with trailing comment\n\
|
||||
invalid-not-ip\n\
|
||||
256.1.1.1\n";
|
||||
|
||||
let entries = parse_feed_body(body);
|
||||
assert_eq!(entries.len(), 3);
|
||||
assert_eq!(entries[0], FeedEntry::Single(Ipv4Addr::new(192, 168, 1, 1)));
|
||||
assert_eq!(entries[1], FeedEntry::Single(Ipv4Addr::new(10, 0, 0, 1)));
|
||||
assert_eq!(entries[2], FeedEntry::Single(Ipv4Addr::new(172, 16, 0, 1)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_cidr_preserves_prefix() {
|
||||
let entries = parse_feed_body("10.0.0.0/8\n192.168.0.0/16\n");
|
||||
assert_eq!(entries.len(), 2);
|
||||
assert_eq!(entries[0], FeedEntry::Cidr(Ipv4Addr::new(10, 0, 0, 0), 8));
|
||||
assert_eq!(entries[1], FeedEntry::Cidr(Ipv4Addr::new(192, 168, 0, 0), 16));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_mixed_ips_and_cidrs() {
|
||||
let body = "# Spamhaus DROP\n\
|
||||
1.2.3.4\n\
|
||||
10.0.0.0/8\n\
|
||||
5.6.7.8\n\
|
||||
192.168.0.0/24\n";
|
||||
let entries = parse_feed_body(body);
|
||||
assert_eq!(entries.len(), 4);
|
||||
assert_eq!(entries[0], FeedEntry::Single(Ipv4Addr::new(1, 2, 3, 4)));
|
||||
assert_eq!(entries[1], FeedEntry::Cidr(Ipv4Addr::new(10, 0, 0, 0), 8));
|
||||
assert_eq!(entries[2], FeedEntry::Single(Ipv4Addr::new(5, 6, 7, 8)));
|
||||
assert_eq!(entries[3], FeedEntry::Cidr(Ipv4Addr::new(192, 168, 0, 0), 24));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_invalid_cidr_prefix_rejected() {
|
||||
let entries = parse_feed_body("10.0.0.0/33\n");
|
||||
assert!(entries.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn feed_source_construction() {
|
||||
let src = FeedSource {
|
||||
name: "test".into(),
|
||||
url: "http://example.com/ips.txt".into(),
|
||||
block_duration_secs: 3600,
|
||||
};
|
||||
assert_eq!(src.block_duration_secs, 3600);
|
||||
}
|
||||
}
|
||||
229
blackwall/src/firewall.rs
Executable file
229
blackwall/src/firewall.rs
Executable file
|
|
@ -0,0 +1,229 @@
|
|||
use anyhow::{Context, Result};
|
||||
use std::collections::HashSet;
|
||||
use std::io::Write;
|
||||
use std::net::Ipv4Addr;
|
||||
use std::process::{Command, Stdio};
|
||||
use std::time::Instant;
|
||||
|
||||
/// Max iptables fork/exec calls per second to prevent resource exhaustion
|
||||
/// under DDoS. Excess IPs are queued and applied via `iptables-restore` batch.
|
||||
const MAX_INDIVIDUAL_OPS_PER_SEC: u32 = 10;
|
||||
|
||||
/// Manages iptables DNAT rules to redirect attacker traffic to the tarpit.
|
||||
///
|
||||
/// # Deprecation
|
||||
/// **V2.0**: eBPF native DNAT via TARPIT_TARGET map replaces iptables.
|
||||
/// This manager is retained as a legacy fallback for systems where XDP
|
||||
/// header rewriting is not available (e.g., HW offload without TC support).
|
||||
/// New deployments should use eBPF DNAT exclusively.
|
||||
///
|
||||
/// # Performance
|
||||
/// Individual `redirect_to_tarpit` calls use fork/exec (fine at low rate).
|
||||
/// Under DDoS, callers should use `redirect_batch` or call `flush_pending`
|
||||
/// periodically — these use `iptables-restore --noflush` (single fork for N rules).
|
||||
#[deprecated(note = "V2.0: eBPF native DNAT replaces iptables. Retained as legacy fallback.")]
|
||||
pub struct FirewallManager {
|
||||
active_redirects: HashSet<Ipv4Addr>,
|
||||
/// IPs awaiting batch insertion (rate-limit overflow).
|
||||
pending_redirects: Vec<Ipv4Addr>,
|
||||
tarpit_port: u16,
|
||||
/// Rate limiter: ops performed in the current second.
|
||||
ops_this_window: u32,
|
||||
/// Start of the current rate-limit window.
|
||||
window_start: Instant,
|
||||
}
|
||||
|
||||
impl FirewallManager {
|
||||
/// Create a new FirewallManager targeting the given tarpit port.
|
||||
pub fn new(tarpit_port: u16) -> Self {
|
||||
Self {
|
||||
active_redirects: HashSet::new(),
|
||||
pending_redirects: Vec::new(),
|
||||
tarpit_port,
|
||||
ops_this_window: 0,
|
||||
window_start: Instant::now(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Reset the rate-limit window if a second has elapsed.
|
||||
fn maybe_reset_window(&mut self) {
|
||||
if self.window_start.elapsed().as_secs() >= 1 {
|
||||
self.ops_this_window = 0;
|
||||
self.window_start = Instant::now();
|
||||
}
|
||||
}
|
||||
|
||||
/// Add a DNAT rule to redirect all TCP traffic from `ip` to the tarpit.
|
||||
///
|
||||
/// If the per-second rate limit is exceeded, the IP is queued for batch
|
||||
/// insertion via `flush_pending()`.
|
||||
pub fn redirect_to_tarpit(&mut self, ip: Ipv4Addr) -> Result<()> {
|
||||
if self.active_redirects.contains(&ip) {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
self.maybe_reset_window();
|
||||
|
||||
if self.ops_this_window >= MAX_INDIVIDUAL_OPS_PER_SEC {
|
||||
// Rate limit hit — queue for batch insertion
|
||||
if !self.pending_redirects.contains(&ip) {
|
||||
self.pending_redirects.push(ip);
|
||||
tracing::debug!(%ip, pending = self.pending_redirects.len(),
|
||||
"DNAT queued (rate limit)");
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
self.apply_single_redirect(ip)?;
|
||||
self.ops_this_window += 1;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Apply a single iptables DNAT rule via fork/exec.
|
||||
fn apply_single_redirect(&mut self, ip: Ipv4Addr) -> Result<()> {
|
||||
let dest = format!("127.0.0.1:{}", self.tarpit_port);
|
||||
let status = Command::new("iptables")
|
||||
.args([
|
||||
"-t", "nat", "-A", "PREROUTING",
|
||||
"-s", &ip.to_string(),
|
||||
"-p", "tcp",
|
||||
"-j", "DNAT",
|
||||
"--to-destination", &dest,
|
||||
])
|
||||
.status()
|
||||
.context("failed to execute iptables")?;
|
||||
|
||||
if !status.success() {
|
||||
anyhow::bail!("iptables returned non-zero status for redirect of {}", ip);
|
||||
}
|
||||
|
||||
self.active_redirects.insert(ip);
|
||||
tracing::info!(%ip, "iptables DNAT redirect added");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Batch-insert multiple DNAT rules in a single `iptables-restore` call.
|
||||
///
|
||||
/// Uses `--noflush` to append rules without clearing existing tables.
|
||||
/// Single fork/exec for N rules — O(1) process creation vs O(N).
|
||||
pub fn redirect_batch(&mut self, ips: &[Ipv4Addr]) -> Result<()> {
|
||||
let new_ips: Vec<Ipv4Addr> = ips
|
||||
.iter()
|
||||
.copied()
|
||||
.filter(|ip| !self.active_redirects.contains(ip))
|
||||
.collect();
|
||||
|
||||
if new_ips.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let dest = format!("127.0.0.1:{}", self.tarpit_port);
|
||||
let mut rules = String::from("*nat\n");
|
||||
for ip in &new_ips {
|
||||
// PERF: single iptables-restore call for all IPs
|
||||
rules.push_str(&format!(
|
||||
"-A PREROUTING -s {} -p tcp -j DNAT --to-destination {}\n",
|
||||
ip, dest
|
||||
));
|
||||
}
|
||||
rules.push_str("COMMIT\n");
|
||||
|
||||
let mut child = Command::new("iptables-restore")
|
||||
.arg("--noflush")
|
||||
.stdin(Stdio::piped())
|
||||
.stdout(Stdio::null())
|
||||
.stderr(Stdio::piped())
|
||||
.spawn()
|
||||
.context("failed to spawn iptables-restore")?;
|
||||
|
||||
if let Some(mut stdin) = child.stdin.take() {
|
||||
stdin
|
||||
.write_all(rules.as_bytes())
|
||||
.context("failed to write to iptables-restore stdin")?;
|
||||
}
|
||||
|
||||
let output = child
|
||||
.wait_with_output()
|
||||
.context("iptables-restore failed")?;
|
||||
|
||||
if !output.status.success() {
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
anyhow::bail!("iptables-restore failed: {}", stderr);
|
||||
}
|
||||
|
||||
for ip in &new_ips {
|
||||
self.active_redirects.insert(*ip);
|
||||
}
|
||||
|
||||
tracing::info!(count = new_ips.len(), "iptables DNAT batch redirect added");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Flush all pending (rate-limited) redirects via a single batch call.
|
||||
///
|
||||
/// Should be called periodically from the event loop (e.g. every 1s).
|
||||
pub fn flush_pending(&mut self) -> Result<()> {
|
||||
if self.pending_redirects.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let pending = std::mem::take(&mut self.pending_redirects);
|
||||
tracing::info!(count = pending.len(), "flushing pending DNAT redirects");
|
||||
self.redirect_batch(&pending)
|
||||
}
|
||||
|
||||
/// Remove the DNAT rule for a specific IP.
|
||||
pub fn remove_redirect(&mut self, ip: Ipv4Addr) -> Result<()> {
|
||||
if !self.active_redirects.contains(&ip) {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let dest = format!("127.0.0.1:{}", self.tarpit_port);
|
||||
let status = Command::new("iptables")
|
||||
.args([
|
||||
"-t", "nat", "-D", "PREROUTING",
|
||||
"-s", &ip.to_string(),
|
||||
"-p", "tcp",
|
||||
"-j", "DNAT",
|
||||
"--to-destination", &dest,
|
||||
])
|
||||
.status()
|
||||
.context("failed to execute iptables")?;
|
||||
|
||||
if !status.success() {
|
||||
tracing::warn!(%ip, "iptables rule removal returned non-zero");
|
||||
}
|
||||
|
||||
self.active_redirects.remove(&ip);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Remove all active redirect rules. Called on graceful shutdown.
|
||||
pub fn cleanup_all(&mut self) -> Result<()> {
|
||||
let ips: Vec<Ipv4Addr> = self.active_redirects.iter().copied().collect();
|
||||
for ip in ips {
|
||||
if let Err(e) = self.remove_redirect(ip) {
|
||||
tracing::warn!(%ip, "cleanup failed: {}", e);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Number of currently active DNAT redirects.
|
||||
pub fn active_count(&self) -> usize {
|
||||
self.active_redirects.len()
|
||||
}
|
||||
|
||||
/// Number of pending (rate-limited) redirects awaiting flush.
|
||||
pub fn pending_count(&self) -> usize {
|
||||
self.pending_redirects.len()
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for FirewallManager {
|
||||
fn drop(&mut self) {
|
||||
if let Err(e) = self.cleanup_all() {
|
||||
tracing::error!("firewall cleanup on drop failed: {}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
230
blackwall/src/ja4/assembler.rs
Executable file
230
blackwall/src/ja4/assembler.rs
Executable file
|
|
@ -0,0 +1,230 @@
|
|||
//! JA4 fingerprint assembler: converts raw TLS ClientHello components
|
||||
//! into a JA4-format string.
|
||||
//!
|
||||
//! JA4 format (simplified):
|
||||
//! `t{version}{sni_flag}{cipher_count}{ext_count}_{cipher_hash}_{ext_hash}`
|
||||
//!
|
||||
//! - version: TLS version code (12=TLS1.2, 13=TLS1.3)
|
||||
//! - sni_flag: 'd' if SNI present, 'i' if absent
|
||||
//! - cipher_count: 2-digit count of cipher suites (capped at 99)
|
||||
//! - ext_count: 2-digit count of extensions (capped at 99)
|
||||
//! - cipher_hash: first 12 chars of hex-encoded hash of sorted cipher suite IDs
|
||||
//! - ext_hash: first 12 chars of hex-encoded hash of sorted extension IDs
|
||||
|
||||
use common::TlsComponentsEvent;
|
||||
|
||||
/// Assembled JA4 fingerprint with metadata.
|
||||
#[derive(Debug, Clone)]
|
||||
#[allow(dead_code)]
|
||||
pub struct Ja4Fingerprint {
|
||||
/// Full JA4 string (e.g., "t13d1510_a0b1c2d3e4f5_f5e4d3c2b1a0")
|
||||
pub fingerprint: String,
|
||||
/// Source IP (network byte order)
|
||||
pub src_ip: u32,
|
||||
/// Destination IP (network byte order)
|
||||
pub dst_ip: u32,
|
||||
/// Source port
|
||||
pub src_port: u16,
|
||||
/// Destination port
|
||||
pub dst_port: u16,
|
||||
/// SNI hostname (if present)
|
||||
pub sni: Option<String>,
|
||||
}
|
||||
|
||||
/// Assembles JA4 fingerprints from eBPF TlsComponentsEvent.
|
||||
pub struct Ja4Assembler;
|
||||
|
||||
impl Ja4Assembler {
|
||||
/// Compute JA4 fingerprint from raw TLS ClientHello components.
|
||||
pub fn assemble(event: &TlsComponentsEvent) -> Ja4Fingerprint {
|
||||
let version = tls_version_code(event.tls_version);
|
||||
let sni_flag = if event.has_sni != 0 { 'd' } else { 'i' };
|
||||
let cipher_count = (event.cipher_count as u16).min(99);
|
||||
let ext_count = (event.ext_count as u16).min(99);
|
||||
|
||||
// Sort and hash cipher suites
|
||||
let mut ciphers: Vec<u16> = event.ciphers[..event.cipher_count as usize]
|
||||
.iter()
|
||||
.copied()
|
||||
// GREASE values: 0x{0a,1a,2a,...,fa}0a — skip them
|
||||
.filter(|&c| !is_grease(c))
|
||||
.collect();
|
||||
ciphers.sort_unstable();
|
||||
let cipher_hash = truncated_hash(&ciphers);
|
||||
|
||||
// Sort and hash extensions
|
||||
let mut extensions: Vec<u16> = event.extensions[..event.ext_count as usize]
|
||||
.iter()
|
||||
.copied()
|
||||
.filter(|&e| !is_grease(e))
|
||||
.collect();
|
||||
extensions.sort_unstable();
|
||||
let ext_hash = truncated_hash(&extensions);
|
||||
|
||||
// Build JA4 string
|
||||
let fingerprint = format!(
|
||||
"t{}{}{:02}{:02}_{}_{}",
|
||||
version, sni_flag, cipher_count, ext_count, cipher_hash, ext_hash
|
||||
);
|
||||
|
||||
// Extract SNI
|
||||
let sni = if event.has_sni != 0 {
|
||||
let sni_bytes = &event.sni[..];
|
||||
let end = sni_bytes.iter().position(|&b| b == 0).unwrap_or(sni_bytes.len());
|
||||
if end > 0 {
|
||||
Some(String::from_utf8_lossy(&sni_bytes[..end]).into_owned())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
Ja4Fingerprint {
|
||||
fingerprint,
|
||||
src_ip: event.src_ip,
|
||||
dst_ip: event.dst_ip,
|
||||
src_port: event.src_port,
|
||||
dst_port: event.dst_port,
|
||||
sni,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Map TLS version u16 to JA4 version code.
|
||||
fn tls_version_code(version: u16) -> &'static str {
|
||||
match version {
|
||||
0x0304 => "13",
|
||||
0x0303 => "12",
|
||||
0x0302 => "11",
|
||||
0x0301 => "10",
|
||||
0x0300 => "s3",
|
||||
_ => "00",
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if a TLS value is a GREASE (Generate Random Extensions And Sustain Extensibility) value.
|
||||
/// GREASE values follow pattern: 0x{0a,1a,2a,...,fa}0a
|
||||
fn is_grease(val: u16) -> bool {
|
||||
let hi = (val >> 8) as u8;
|
||||
let lo = val as u8;
|
||||
lo == 0x0a && hi & 0x0f == 0x0a
|
||||
}
|
||||
|
||||
/// Compute a simple hash of sorted u16 values, return first 12 hex chars.
|
||||
/// Uses FNV-1a for speed (no cryptographic requirement).
|
||||
fn truncated_hash(values: &[u16]) -> String {
|
||||
let mut hash: u64 = 0xcbf29ce484222325; // FNV offset basis
|
||||
for &v in values {
|
||||
let bytes = v.to_be_bytes();
|
||||
for &b in &bytes {
|
||||
hash ^= b as u64;
|
||||
hash = hash.wrapping_mul(0x100000001b3); // FNV prime
|
||||
}
|
||||
}
|
||||
format!("{:012x}", hash)[..12].to_string()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn make_tls_event(
|
||||
version: u16,
|
||||
ciphers: &[u16],
|
||||
extensions: &[u16],
|
||||
has_sni: bool,
|
||||
sni: &[u8],
|
||||
) -> TlsComponentsEvent {
|
||||
let mut event = TlsComponentsEvent {
|
||||
src_ip: 0x0100007f,
|
||||
dst_ip: 0xC0A80001u32.to_be(),
|
||||
src_port: 54321,
|
||||
dst_port: 443,
|
||||
tls_version: version,
|
||||
cipher_count: ciphers.len().min(20) as u8,
|
||||
ext_count: extensions.len().min(20) as u8,
|
||||
ciphers: [0u16; 20],
|
||||
extensions: [0u16; 20],
|
||||
sni: [0u8; 32],
|
||||
alpn_first_len: 0,
|
||||
has_sni: if has_sni { 1 } else { 0 },
|
||||
timestamp_ns: 0,
|
||||
_padding: [0; 2],
|
||||
};
|
||||
for (i, &c) in ciphers.iter().take(20).enumerate() {
|
||||
event.ciphers[i] = c;
|
||||
}
|
||||
for (i, &e) in extensions.iter().take(20).enumerate() {
|
||||
event.extensions[i] = e;
|
||||
}
|
||||
let copy_len = sni.len().min(32);
|
||||
event.sni[..copy_len].copy_from_slice(&sni[..copy_len]);
|
||||
event
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ja4_tls13_with_sni() {
|
||||
let event = make_tls_event(
|
||||
0x0304,
|
||||
&[0x1301, 0x1302, 0x1303],
|
||||
&[0x0000, 0x000a, 0x000b, 0x000d],
|
||||
true,
|
||||
b"example.com",
|
||||
);
|
||||
let fp = Ja4Assembler::assemble(&event);
|
||||
assert!(fp.fingerprint.starts_with("t13d0304_"));
|
||||
assert_eq!(fp.sni, Some("example.com".to_string()));
|
||||
assert_eq!(fp.dst_port, 443);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ja4_tls12_no_sni() {
|
||||
let event = make_tls_event(
|
||||
0x0303,
|
||||
&[0xc02c, 0xc02b, 0x009e],
|
||||
&[0x000a, 0x000b],
|
||||
false,
|
||||
&[],
|
||||
);
|
||||
let fp = Ja4Assembler::assemble(&event);
|
||||
assert!(fp.fingerprint.starts_with("t12i0302_"));
|
||||
assert_eq!(fp.sni, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn grease_values_filtered() {
|
||||
// 0x0a0a is a GREASE value
|
||||
assert!(is_grease(0x0a0a));
|
||||
assert!(is_grease(0x1a0a));
|
||||
assert!(is_grease(0xfa0a));
|
||||
assert!(!is_grease(0x0001));
|
||||
assert!(!is_grease(0x1301));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truncated_hash_deterministic() {
|
||||
let h1 = truncated_hash(&[0x1301, 0x1302, 0x1303]);
|
||||
let h2 = truncated_hash(&[0x1301, 0x1302, 0x1303]);
|
||||
assert_eq!(h1, h2);
|
||||
assert_eq!(h1.len(), 12);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truncated_hash_order_matters() {
|
||||
// Input is pre-sorted, so different order = different hash
|
||||
let h1 = truncated_hash(&[0x0001, 0x0002]);
|
||||
let h2 = truncated_hash(&[0x0002, 0x0001]);
|
||||
assert_ne!(h1, h2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tls_version_mapping() {
|
||||
assert_eq!(tls_version_code(0x0304), "13");
|
||||
assert_eq!(tls_version_code(0x0303), "12");
|
||||
assert_eq!(tls_version_code(0x0302), "11");
|
||||
assert_eq!(tls_version_code(0x0301), "10");
|
||||
assert_eq!(tls_version_code(0x0300), "s3");
|
||||
assert_eq!(tls_version_code(0x0200), "00");
|
||||
}
|
||||
}
|
||||
167
blackwall/src/ja4/db.rs
Executable file
167
blackwall/src/ja4/db.rs
Executable file
|
|
@ -0,0 +1,167 @@
|
|||
//! JA4 fingerprint database: known fingerprint matching.
|
||||
//!
|
||||
//! Maintains a HashMap of known JA4 fingerprints to tool/client names.
|
||||
//! Can be populated from a static list or loaded from a config file.
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
/// Result of matching a JA4 fingerprint against the database.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum Ja4Match {
|
||||
/// Known malicious tool.
|
||||
Malicious { name: String, confidence: f32 },
|
||||
/// Known legitimate client.
|
||||
Benign { name: String },
|
||||
/// No match in database.
|
||||
Unknown,
|
||||
}
|
||||
|
||||
/// Database of known JA4 fingerprints.
|
||||
pub struct Ja4Database {
|
||||
/// Maps JA4 fingerprint prefix (first segment before underscore) → entries.
|
||||
entries: HashMap<String, Ja4Entry>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct Ja4Entry {
|
||||
name: String,
|
||||
is_malicious: bool,
|
||||
confidence: f32,
|
||||
}
|
||||
|
||||
impl Ja4Database {
|
||||
/// Create an empty database.
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
entries: HashMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a database pre-populated with common known fingerprints.
|
||||
pub fn with_defaults() -> Self {
|
||||
let mut db = Self::new();
|
||||
|
||||
// Known scanning tools
|
||||
db.add_malicious("t13d0103", "nmap", 0.9);
|
||||
db.add_malicious("t12i0003", "masscan", 0.85);
|
||||
db.add_malicious("t12i0103", "zgrab2", 0.8);
|
||||
db.add_malicious("t13d0203", "nuclei", 0.85);
|
||||
db.add_malicious("t12d0305", "sqlmap", 0.9);
|
||||
db.add_malicious("t13d0105", "gobuster", 0.8);
|
||||
|
||||
// Known legitimate clients
|
||||
db.add_benign("t13d1510", "Chrome/modern");
|
||||
db.add_benign("t13d1609", "Firefox/modern");
|
||||
db.add_benign("t13d0907", "Safari/modern");
|
||||
db.add_benign("t13d1208", "Edge/modern");
|
||||
db.add_benign("t13d0605", "curl");
|
||||
db.add_benign("t13d0404", "python-requests");
|
||||
|
||||
db
|
||||
}
|
||||
|
||||
/// Add a known malicious fingerprint.
|
||||
pub fn add_malicious(&mut self, prefix: &str, name: &str, confidence: f32) {
|
||||
self.entries.insert(
|
||||
prefix.to_string(),
|
||||
Ja4Entry {
|
||||
name: name.to_string(),
|
||||
is_malicious: true,
|
||||
confidence,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
/// Add a known benign fingerprint.
|
||||
pub fn add_benign(&mut self, prefix: &str, name: &str) {
|
||||
self.entries.insert(
|
||||
prefix.to_string(),
|
||||
Ja4Entry {
|
||||
name: name.to_string(),
|
||||
is_malicious: false,
|
||||
confidence: 0.0,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
/// Look up a JA4 fingerprint. Matches on the first segment (before first '_').
|
||||
pub fn lookup(&self, fingerprint: &str) -> Ja4Match {
|
||||
let prefix = fingerprint
|
||||
.split('_')
|
||||
.next()
|
||||
.unwrap_or(fingerprint);
|
||||
|
||||
match self.entries.get(prefix) {
|
||||
Some(entry) if entry.is_malicious => Ja4Match::Malicious {
|
||||
name: entry.name.clone(),
|
||||
confidence: entry.confidence,
|
||||
},
|
||||
Some(entry) => Ja4Match::Benign {
|
||||
name: entry.name.clone(),
|
||||
},
|
||||
None => Ja4Match::Unknown,
|
||||
}
|
||||
}
|
||||
|
||||
/// Number of entries in the database.
|
||||
#[allow(dead_code)]
|
||||
pub fn len(&self) -> usize {
|
||||
self.entries.len()
|
||||
}
|
||||
|
||||
/// Whether the database is empty.
|
||||
#[allow(dead_code)]
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.entries.is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn empty_database_returns_unknown() {
|
||||
let db = Ja4Database::new();
|
||||
assert_eq!(db.lookup("t13d1510_abc123_def456"), Ja4Match::Unknown);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn defaults_include_known_tools() {
|
||||
let db = Ja4Database::with_defaults();
|
||||
assert!(db.len() > 0);
|
||||
|
||||
match db.lookup("t13d0103_anything_here") {
|
||||
Ja4Match::Malicious { name, .. } => assert_eq!(name, "nmap"),
|
||||
other => panic!("expected nmap match, got {:?}", other),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn benign_lookup() {
|
||||
let db = Ja4Database::with_defaults();
|
||||
match db.lookup("t13d1510_hash1_hash2") {
|
||||
Ja4Match::Benign { name } => assert_eq!(name, "Chrome/modern"),
|
||||
other => panic!("expected Chrome match, got {:?}", other),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_fingerprint() {
|
||||
let db = Ja4Database::with_defaults();
|
||||
assert_eq!(db.lookup("t13d9999_unknown_hash"), Ja4Match::Unknown);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn custom_entries() {
|
||||
let mut db = Ja4Database::new();
|
||||
db.add_malicious("t12i0201", "custom_scanner", 0.75);
|
||||
match db.lookup("t12i0201_hash_hash") {
|
||||
Ja4Match::Malicious { name, confidence } => {
|
||||
assert_eq!(name, "custom_scanner");
|
||||
assert!((confidence - 0.75).abs() < 0.01);
|
||||
}
|
||||
other => panic!("expected custom scanner, got {:?}", other),
|
||||
}
|
||||
}
|
||||
}
|
||||
12
blackwall/src/ja4/mod.rs
Executable file
12
blackwall/src/ja4/mod.rs
Executable file
|
|
@ -0,0 +1,12 @@
|
|||
//! JA4 TLS fingerprinting module.
|
||||
//!
|
||||
//! Assembles JA4 fingerprints from raw TLS ClientHello components
|
||||
//! emitted by the eBPF program. JA4 format:
|
||||
//! `t{TLSver}{SNI}{CipherCount}{ExtCount}_{CipherHash}_{ExtHash}`
|
||||
//!
|
||||
//! Reference: https://github.com/FoxIO-LLC/ja4
|
||||
|
||||
// ARCH: JA4 module is wired into the main event loop via TLS_EVENTS RingBuf.
|
||||
// eBPF TLS ClientHello parser emits TlsComponentsEvent → JA4 assembly → DB lookup.
|
||||
pub mod assembler;
|
||||
pub mod db;
|
||||
1223
blackwall/src/main.rs
Executable file
1223
blackwall/src/main.rs
Executable file
File diff suppressed because it is too large
Load diff
97
blackwall/src/metrics.rs
Executable file
97
blackwall/src/metrics.rs
Executable file
|
|
@ -0,0 +1,97 @@
|
|||
use anyhow::Result;
|
||||
use aya::maps::{MapData, PerCpuArray};
|
||||
use common::Counters;
|
||||
use http_body_util::Full;
|
||||
use hyper::body::Bytes;
|
||||
use hyper::Request;
|
||||
use hyper_util::rt::TokioIo;
|
||||
use std::time::Duration;
|
||||
|
||||
const HIVEMIND_PUSH_URL: &str = "http://127.0.0.1:8090/push";
|
||||
|
||||
/// Periodically read eBPF COUNTERS (sum across CPUs), log via tracing,
|
||||
/// and push the latest values to hivemind-api for the live dashboard.
|
||||
pub async fn metrics_tick(
|
||||
counters: PerCpuArray<MapData, Counters>,
|
||||
interval_secs: u64,
|
||||
) -> Result<()> {
|
||||
let mut interval = tokio::time::interval(Duration::from_secs(interval_secs));
|
||||
|
||||
loop {
|
||||
interval.tick().await;
|
||||
|
||||
let values = match counters.get(&0, 0) {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
tracing::warn!("failed to read counters: {}", e);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
let total = values.iter().fold(
|
||||
Counters {
|
||||
packets_total: 0,
|
||||
packets_passed: 0,
|
||||
packets_dropped: 0,
|
||||
anomalies_sent: 0,
|
||||
},
|
||||
|acc, c| Counters {
|
||||
packets_total: acc.packets_total + c.packets_total,
|
||||
packets_passed: acc.packets_passed + c.packets_passed,
|
||||
packets_dropped: acc.packets_dropped + c.packets_dropped,
|
||||
anomalies_sent: acc.anomalies_sent + c.anomalies_sent,
|
||||
},
|
||||
);
|
||||
|
||||
tracing::info!(
|
||||
total = total.packets_total,
|
||||
passed = total.packets_passed,
|
||||
dropped = total.packets_dropped,
|
||||
anomalies = total.anomalies_sent,
|
||||
"counters"
|
||||
);
|
||||
|
||||
// Push live metrics to hivemind-api dashboard
|
||||
push_to_hivemind(
|
||||
total.packets_total,
|
||||
total.packets_passed,
|
||||
total.packets_dropped,
|
||||
total.anomalies_sent,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Fire-and-forget push of current eBPF counter totals to hivemind-api.
|
||||
async fn push_to_hivemind(
|
||||
packets_total: u64,
|
||||
packets_passed: u64,
|
||||
packets_dropped: u64,
|
||||
anomalies_sent: u64,
|
||||
) {
|
||||
let body = format!(
|
||||
r#"{{"packets_total":{packets_total},"packets_passed":{packets_passed},"packets_dropped":{packets_dropped},"anomalies_sent":{anomalies_sent}}}"#
|
||||
);
|
||||
let result = async {
|
||||
let stream = tokio::net::TcpStream::connect("127.0.0.1:8090").await?;
|
||||
let io = TokioIo::new(stream);
|
||||
let (mut sender, conn) = hyper::client::conn::http1::handshake(io).await?;
|
||||
tokio::spawn(async move {
|
||||
if let Err(e) = conn.await {
|
||||
tracing::debug!(error = %e, "hivemind-api push connection dropped");
|
||||
}
|
||||
});
|
||||
let req = Request::builder()
|
||||
.method("POST")
|
||||
.uri(HIVEMIND_PUSH_URL)
|
||||
.header("content-type", "application/json")
|
||||
.header("host", "127.0.0.1")
|
||||
.body(Full::new(Bytes::from(body)))?;
|
||||
sender.send_request(req).await?;
|
||||
anyhow::Ok(())
|
||||
}
|
||||
.await;
|
||||
if let Err(e) = result {
|
||||
tracing::debug!(error = %e, "failed to push metrics to hivemind-api (daemon may not be running)");
|
||||
}
|
||||
}
|
||||
297
blackwall/src/pcap.rs
Executable file
297
blackwall/src/pcap.rs
Executable file
|
|
@ -0,0 +1,297 @@
|
|||
//! PCAP file writer for forensic packet capture.
|
||||
//!
|
||||
//! Writes pcap-format files for flagged IPs. Uses the standard pcap file
|
||||
//! format (libpcap) without any external dependencies.
|
||||
|
||||
#![allow(dead_code)]
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use std::collections::HashSet;
|
||||
use std::io::Write;
|
||||
use std::net::Ipv4Addr;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Mutex;
|
||||
|
||||
/// PCAP global header magic number (microsecond resolution).
|
||||
const PCAP_MAGIC: u32 = 0xa1b2c3d4;
|
||||
/// PCAP version 2.4.
|
||||
const PCAP_VERSION_MAJOR: u16 = 2;
|
||||
const PCAP_VERSION_MINOR: u16 = 4;
|
||||
/// Maximum bytes to capture per packet.
|
||||
const SNAP_LEN: u32 = 65535;
|
||||
/// Link type: raw IPv4 (DLT_RAW = 228). Alternative: Ethernet = 1.
|
||||
const LINK_TYPE_RAW: u32 = 228;
|
||||
|
||||
/// Maximum PCAP file size before rotation (100 MB).
|
||||
const MAX_FILE_SIZE: u64 = 100 * 1024 * 1024;
|
||||
/// Maximum number of rotated files to keep.
|
||||
const MAX_ROTATED_FILES: usize = 10;
|
||||
|
||||
/// Manages PCAP capture for flagged IPs.
|
||||
pub struct PcapWriter {
|
||||
/// Directory to write pcap files.
|
||||
output_dir: PathBuf,
|
||||
/// Set of IPs currently flagged for capture.
|
||||
flagged_ips: Mutex<HashSet<u32>>,
|
||||
/// Currently open pcap file (if any).
|
||||
file: Mutex<Option<PcapFile>>,
|
||||
}
|
||||
|
||||
struct PcapFile {
|
||||
writer: std::io::BufWriter<std::fs::File>,
|
||||
path: PathBuf,
|
||||
bytes_written: u64,
|
||||
}
|
||||
|
||||
impl PcapWriter {
|
||||
/// Create a new PCAP writer with the given output directory.
|
||||
pub fn new(output_dir: PathBuf) -> Result<Self> {
|
||||
std::fs::create_dir_all(&output_dir)
|
||||
.with_context(|| format!("failed to create pcap dir: {}", output_dir.display()))?;
|
||||
Ok(Self {
|
||||
output_dir,
|
||||
flagged_ips: Mutex::new(HashSet::new()),
|
||||
file: Mutex::new(None),
|
||||
})
|
||||
}
|
||||
|
||||
/// Flag an IP for packet capture.
|
||||
pub fn flag_ip(&self, ip: Ipv4Addr) {
|
||||
let raw = u32::from(ip);
|
||||
let mut ips = self.flagged_ips.lock().expect("flagged_ips lock");
|
||||
if ips.insert(raw) {
|
||||
tracing::info!(%ip, "PCAP capture enabled for IP");
|
||||
}
|
||||
}
|
||||
|
||||
/// Remove an IP from capture.
|
||||
#[allow(dead_code)]
|
||||
pub fn unflag_ip(&self, ip: Ipv4Addr) {
|
||||
let raw = u32::from(ip);
|
||||
let mut ips = self.flagged_ips.lock().expect("flagged_ips lock");
|
||||
if ips.remove(&raw) {
|
||||
tracing::info!(%ip, "PCAP capture disabled for IP");
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if an IP is flagged for capture.
|
||||
pub fn is_flagged(&self, src_ip: u32) -> bool {
|
||||
let ips = self.flagged_ips.lock().expect("flagged_ips lock");
|
||||
ips.contains(&src_ip)
|
||||
}
|
||||
|
||||
/// Write a raw IP packet to the pcap file (if the IP is flagged).
|
||||
pub fn write_packet(&self, src_ip: u32, dst_ip: u32, data: &[u8]) -> Result<()> {
|
||||
// Check if either endpoint is flagged
|
||||
let ips = self.flagged_ips.lock().expect("flagged_ips lock");
|
||||
if !ips.contains(&src_ip) && !ips.contains(&dst_ip) {
|
||||
return Ok(());
|
||||
}
|
||||
drop(ips); // Release lock before I/O
|
||||
|
||||
let mut file_guard = self.file.lock().expect("pcap file lock");
|
||||
|
||||
// Open file if needed, or rotate if too large
|
||||
let pcap = match file_guard.as_mut() {
|
||||
Some(f) if f.bytes_written < MAX_FILE_SIZE => f,
|
||||
Some(_) => {
|
||||
// Rotate
|
||||
let old = file_guard.take().expect("just checked Some");
|
||||
drop(old);
|
||||
self.rotate_files()?;
|
||||
let new_file = self.open_new_file()?;
|
||||
*file_guard = Some(new_file);
|
||||
file_guard.as_mut().expect("just created")
|
||||
}
|
||||
None => {
|
||||
let new_file = self.open_new_file()?;
|
||||
*file_guard = Some(new_file);
|
||||
file_guard.as_mut().expect("just created")
|
||||
}
|
||||
};
|
||||
|
||||
// Write pcap packet record
|
||||
let ts = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap_or_default();
|
||||
let ts_sec = ts.as_secs() as u32;
|
||||
let ts_usec = ts.subsec_micros();
|
||||
let cap_len = data.len().min(SNAP_LEN as usize) as u32;
|
||||
|
||||
// Packet header: ts_sec(4) + ts_usec(4) + cap_len(4) + orig_len(4)
|
||||
pcap.writer.write_all(&ts_sec.to_le_bytes())?;
|
||||
pcap.writer.write_all(&ts_usec.to_le_bytes())?;
|
||||
pcap.writer.write_all(&cap_len.to_le_bytes())?;
|
||||
pcap.writer.write_all(&(data.len() as u32).to_le_bytes())?;
|
||||
pcap.writer.write_all(&data[..cap_len as usize])?;
|
||||
pcap.writer.flush()?;
|
||||
|
||||
pcap.bytes_written += 16 + cap_len as u64;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Open a new pcap file with a global header.
|
||||
fn open_new_file(&self) -> Result<PcapFile> {
|
||||
let timestamp = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs();
|
||||
let path = self.output_dir.join(format!("capture_{}.pcap", timestamp));
|
||||
|
||||
let file = std::fs::File::create(&path)
|
||||
.with_context(|| format!("failed to create pcap: {}", path.display()))?;
|
||||
let mut writer = std::io::BufWriter::new(file);
|
||||
|
||||
// Write pcap global header
|
||||
writer.write_all(&PCAP_MAGIC.to_le_bytes())?;
|
||||
writer.write_all(&PCAP_VERSION_MAJOR.to_le_bytes())?;
|
||||
writer.write_all(&PCAP_VERSION_MINOR.to_le_bytes())?;
|
||||
writer.write_all(&0i32.to_le_bytes())?; // thiszone
|
||||
writer.write_all(&0u32.to_le_bytes())?; // sigfigs
|
||||
writer.write_all(&SNAP_LEN.to_le_bytes())?;
|
||||
writer.write_all(&LINK_TYPE_RAW.to_le_bytes())?;
|
||||
writer.flush()?;
|
||||
|
||||
tracing::info!(path = %path.display(), "opened new PCAP file");
|
||||
|
||||
Ok(PcapFile {
|
||||
writer,
|
||||
path,
|
||||
bytes_written: 24, // Global header size
|
||||
})
|
||||
}
|
||||
|
||||
/// Rotate pcap files: remove oldest if exceeding MAX_ROTATED_FILES.
|
||||
fn rotate_files(&self) -> Result<()> {
|
||||
let mut entries: Vec<PathBuf> = std::fs::read_dir(&self.output_dir)?
|
||||
.filter_map(|e| e.ok())
|
||||
.filter(|e| {
|
||||
e.path()
|
||||
.extension()
|
||||
.map(|ext| ext == "pcap" || ext == "gz")
|
||||
.unwrap_or(false)
|
||||
})
|
||||
.map(|e| e.path())
|
||||
.collect();
|
||||
|
||||
entries.sort();
|
||||
|
||||
while entries.len() >= MAX_ROTATED_FILES {
|
||||
if let Some(oldest) = entries.first() {
|
||||
tracing::info!(path = %oldest.display(), "rotating old PCAP file");
|
||||
std::fs::remove_file(oldest)?;
|
||||
entries.remove(0);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Compress a pcap file using gzip. Returns path to compressed file.
|
||||
pub fn compress_file(path: &std::path::Path) -> Result<PathBuf> {
|
||||
use std::process::Command;
|
||||
let output = Command::new("gzip")
|
||||
.arg("-f") // Force overwrite
|
||||
.arg(path.as_os_str())
|
||||
.output()
|
||||
.with_context(|| format!("failed to run gzip on {}", path.display()))?;
|
||||
|
||||
if !output.status.success() {
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
anyhow::bail!("gzip failed: {}", stderr);
|
||||
}
|
||||
|
||||
let gz_path = path.with_extension("pcap.gz");
|
||||
Ok(gz_path)
|
||||
}
|
||||
|
||||
/// Get the number of flagged IPs.
|
||||
#[allow(dead_code)]
|
||||
pub fn flagged_count(&self) -> usize {
|
||||
self.flagged_ips.lock().expect("flagged_ips lock").len()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn pcap_global_header_size() {
|
||||
// PCAP global header is exactly 24 bytes
|
||||
let size = 4 + 2 + 2 + 4 + 4 + 4 + 4; // magic + ver_maj + ver_min + tz + sigfigs + snaplen + link
|
||||
assert_eq!(size, 24);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn flag_and_check_ip() {
|
||||
let dir = std::env::temp_dir().join("blackwall_pcap_test");
|
||||
let writer = PcapWriter::new(dir.clone()).unwrap();
|
||||
|
||||
let ip = Ipv4Addr::new(192, 168, 1, 1);
|
||||
assert!(!writer.is_flagged(u32::from(ip)));
|
||||
|
||||
writer.flag_ip(ip);
|
||||
assert!(writer.is_flagged(u32::from(ip)));
|
||||
|
||||
writer.unflag_ip(ip);
|
||||
assert!(!writer.is_flagged(u32::from(ip)));
|
||||
|
||||
// Cleanup
|
||||
let _ = std::fs::remove_dir_all(dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn write_and_verify_pcap() {
|
||||
let dir = std::env::temp_dir().join("blackwall_pcap_write_test");
|
||||
let writer = PcapWriter::new(dir.clone()).unwrap();
|
||||
|
||||
let src_ip = Ipv4Addr::new(10, 0, 0, 1);
|
||||
writer.flag_ip(src_ip);
|
||||
|
||||
// Fake IP packet (just some bytes)
|
||||
let packet = [0x45, 0x00, 0x00, 0x28, 0x00, 0x01, 0x00, 0x00,
|
||||
0x40, 0x06, 0x00, 0x00, 0x0a, 0x00, 0x00, 0x01,
|
||||
0xc0, 0xa8, 0x01, 0x01];
|
||||
|
||||
writer.write_packet(u32::from(src_ip), u32::from(Ipv4Addr::new(192, 168, 1, 1)), &packet).unwrap();
|
||||
|
||||
// Verify file was created
|
||||
let files: Vec<_> = std::fs::read_dir(&dir)
|
||||
.unwrap()
|
||||
.filter_map(|e| e.ok())
|
||||
.filter(|e| e.path().extension().map(|ext| ext == "pcap").unwrap_or(false))
|
||||
.collect();
|
||||
assert_eq!(files.len(), 1);
|
||||
|
||||
// Verify file starts with pcap magic
|
||||
let content = std::fs::read(files[0].path()).unwrap();
|
||||
assert!(content.len() >= 24); // At least global header
|
||||
let magic = u32::from_le_bytes([content[0], content[1], content[2], content[3]]);
|
||||
assert_eq!(magic, PCAP_MAGIC);
|
||||
|
||||
// Cleanup
|
||||
let _ = std::fs::remove_dir_all(dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unflagged_ip_not_captured() {
|
||||
let dir = std::env::temp_dir().join("blackwall_pcap_skip_test");
|
||||
let writer = PcapWriter::new(dir.clone()).unwrap();
|
||||
|
||||
let packet = [0x45, 0x00];
|
||||
// No IPs flagged — should be a no-op
|
||||
writer.write_packet(0x0a000001, 0xc0a80101, &packet).unwrap();
|
||||
|
||||
let files: Vec<_> = std::fs::read_dir(&dir)
|
||||
.unwrap()
|
||||
.filter_map(|e| e.ok())
|
||||
.filter(|e| e.path().extension().map(|ext| ext == "pcap").unwrap_or(false))
|
||||
.collect();
|
||||
assert!(files.is_empty());
|
||||
|
||||
// Cleanup
|
||||
let _ = std::fs::remove_dir_all(dir);
|
||||
}
|
||||
}
|
||||
156
blackwall/src/rules.rs
Executable file
156
blackwall/src/rules.rs
Executable file
|
|
@ -0,0 +1,156 @@
|
|||
use anyhow::{Context, Result};
|
||||
use aya::maps::{HashMap, LpmTrie, MapData};
|
||||
use common::{RuleAction, RuleKey, RuleValue};
|
||||
|
||||
/// Manages eBPF maps for IP blocklist, CIDR rules, and expiry.
|
||||
pub struct RuleManager {
|
||||
blocklist: HashMap<MapData, RuleKey, RuleValue>,
|
||||
#[allow(dead_code)]
|
||||
cidr_rules: LpmTrie<MapData, u32, RuleValue>,
|
||||
}
|
||||
|
||||
impl RuleManager {
|
||||
/// Create a new RuleManager from opened eBPF maps.
|
||||
pub fn new(
|
||||
blocklist: HashMap<MapData, RuleKey, RuleValue>,
|
||||
cidr_rules: LpmTrie<MapData, u32, RuleValue>,
|
||||
) -> Self {
|
||||
Self {
|
||||
blocklist,
|
||||
cidr_rules,
|
||||
}
|
||||
}
|
||||
|
||||
/// Block an IP for `duration_secs` seconds (0 = permanent).
|
||||
pub fn block_ip(&mut self, ip: u32, duration_secs: u32) -> Result<()> {
|
||||
let key = RuleKey { ip };
|
||||
let value = RuleValue {
|
||||
action: RuleAction::Drop as u8,
|
||||
_pad1: 0,
|
||||
_pad2: 0,
|
||||
expires_at: if duration_secs == 0 {
|
||||
0
|
||||
} else {
|
||||
current_boot_secs() + duration_secs
|
||||
},
|
||||
};
|
||||
self.blocklist
|
||||
.insert(key, value, 0)
|
||||
.context("failed to insert blocklist entry")?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Explicitly allow an IP (permanent).
|
||||
pub fn allow_ip(&mut self, ip: u32) -> Result<()> {
|
||||
let key = RuleKey { ip };
|
||||
let value = RuleValue {
|
||||
action: RuleAction::Pass as u8,
|
||||
_pad1: 0,
|
||||
_pad2: 0,
|
||||
expires_at: 0,
|
||||
};
|
||||
self.blocklist
|
||||
.insert(key, value, 0)
|
||||
.context("failed to insert allow entry")?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Redirect an IP to the tarpit for `duration_secs`.
|
||||
pub fn redirect_to_tarpit(&mut self, ip: u32, duration_secs: u32) -> Result<()> {
|
||||
let key = RuleKey { ip };
|
||||
let value = RuleValue {
|
||||
action: RuleAction::RedirectTarpit as u8,
|
||||
_pad1: 0,
|
||||
_pad2: 0,
|
||||
expires_at: if duration_secs == 0 {
|
||||
0
|
||||
} else {
|
||||
current_boot_secs() + duration_secs
|
||||
},
|
||||
};
|
||||
self.blocklist
|
||||
.insert(key, value, 0)
|
||||
.context("failed to insert tarpit redirect")?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Check if an IP has a non-pass action (Drop or RedirectTarpit) in the blocklist.
|
||||
pub fn is_blocked_or_redirected(&self, ip: u32) -> bool {
|
||||
let key = RuleKey { ip };
|
||||
match self.blocklist.get(&key, 0) {
|
||||
Ok(v) => v.action != RuleAction::Pass as u8,
|
||||
Err(_) => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Remove an IP from the blocklist.
|
||||
#[allow(dead_code)]
|
||||
pub fn remove_ip(&mut self, ip: u32) -> Result<()> {
|
||||
let key = RuleKey { ip };
|
||||
self.blocklist
|
||||
.remove(&key)
|
||||
.context("failed to remove blocklist entry")?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Add a CIDR rule (e.g., block 10.0.0.0/8).
|
||||
pub fn add_cidr_rule(&mut self, ip: u32, prefix: u32, action: RuleAction) -> Result<()> {
|
||||
let lpm_key = aya::maps::lpm_trie::Key::new(prefix, ip);
|
||||
let value = RuleValue {
|
||||
action: action as u8,
|
||||
_pad1: 0,
|
||||
_pad2: 0,
|
||||
expires_at: 0,
|
||||
};
|
||||
self.cidr_rules
|
||||
.insert(&lpm_key, value, 0)
|
||||
.context("failed to insert CIDR rule")?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Remove expired rules from the blocklist. Returns expired IPs.
|
||||
pub fn expire_stale_rules(&mut self) -> Result<Vec<u32>> {
|
||||
let now = current_boot_secs();
|
||||
let mut expired_keys = Vec::new();
|
||||
|
||||
// Collect keys to expire
|
||||
for result in self.blocklist.iter() {
|
||||
let (key, value) = result.context("error iterating blocklist")?;
|
||||
if value.expires_at != 0 && value.expires_at < now {
|
||||
expired_keys.push(key);
|
||||
}
|
||||
}
|
||||
|
||||
let expired_ips: Vec<u32> = expired_keys.iter().map(|k| k.ip).collect();
|
||||
for key in expired_keys {
|
||||
let _ = self.blocklist.remove(&key);
|
||||
}
|
||||
|
||||
Ok(expired_ips)
|
||||
}
|
||||
|
||||
/// Remove all CIDR rules. Called before feed refresh to avoid stale entries.
|
||||
pub fn clear_cidr_rules(&mut self) -> Result<()> {
|
||||
let keys: Vec<_> = self
|
||||
.cidr_rules
|
||||
.iter()
|
||||
.filter_map(|r| r.ok())
|
||||
.map(|(k, _)| k)
|
||||
.collect();
|
||||
for key in keys {
|
||||
let _ = self.cidr_rules.remove(&key);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Approximate seconds since boot using CLOCK_BOOTTIME.
|
||||
fn current_boot_secs() -> u32 {
|
||||
let mut ts = nix::libc::timespec {
|
||||
tv_sec: 0,
|
||||
tv_nsec: 0,
|
||||
};
|
||||
// SAFETY: valid pointer, CLOCK_BOOTTIME is a valid clock_id
|
||||
unsafe { nix::libc::clock_gettime(nix::libc::CLOCK_BOOTTIME, &mut ts) };
|
||||
ts.tv_sec as u32
|
||||
}
|
||||
389
blackwall/tests/peer_integration.rs
Executable file
389
blackwall/tests/peer_integration.rs
Executable file
|
|
@ -0,0 +1,389 @@
|
|||
//! Integration tests for the distributed peer protocol.
|
||||
//!
|
||||
//! Tests persistent TCP connections, broadcast reuse, and heartbeat cycle.
|
||||
//! Run: `cargo test -p blackwall --test peer_integration -- --nocapture`
|
||||
|
||||
use std::net::{Ipv4Addr, SocketAddr};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use tokio::net::TcpListener;
|
||||
use tokio::sync::Mutex;
|
||||
|
||||
use ring::hmac;
|
||||
|
||||
// Re-import what we need from the crate
|
||||
// NOTE: These tests exercise the wire protocol directly since peer internals
|
||||
// are private. They verify the protocol codec + flow end-to-end.
|
||||
|
||||
/// Wire protocol magic.
|
||||
const PROTOCOL_MAGIC: [u8; 4] = [0x42, 0x57, 0x4C, 0x01];
|
||||
const HELLO_TYPE: u8 = 0x01;
|
||||
const BLOCKED_IP_TYPE: u8 = 0x02;
|
||||
const HEARTBEAT_TYPE: u8 = 0x04;
|
||||
const HMAC_SIZE: usize = 32;
|
||||
const HEADER_SIZE: usize = 4 + 1 + 4 + HMAC_SIZE; // 41
|
||||
|
||||
/// Shared test PSK.
|
||||
fn test_key() -> hmac::Key {
|
||||
hmac::Key::new(hmac::HMAC_SHA256, b"integration-test-psk-blackwall")
|
||||
}
|
||||
|
||||
/// Encode a message using the V2 wire protocol:
|
||||
/// magic(4) + type(1) + len(4) + hmac(32) + payload.
|
||||
fn encode_message(msg_type: u8, payload: &[u8], key: &hmac::Key) -> Vec<u8> {
|
||||
let len = payload.len() as u32;
|
||||
let mut buf = Vec::with_capacity(HEADER_SIZE + payload.len());
|
||||
buf.extend_from_slice(&PROTOCOL_MAGIC);
|
||||
buf.push(msg_type);
|
||||
buf.extend_from_slice(&len.to_le_bytes());
|
||||
|
||||
// Compute HMAC over magic + type + len + payload
|
||||
let mut signing_ctx = hmac::Context::with_key(key);
|
||||
signing_ctx.update(&PROTOCOL_MAGIC);
|
||||
signing_ctx.update(&[msg_type]);
|
||||
signing_ctx.update(&len.to_le_bytes());
|
||||
signing_ctx.update(payload);
|
||||
let tag = signing_ctx.sign();
|
||||
buf.extend_from_slice(tag.as_ref());
|
||||
|
||||
buf.extend_from_slice(payload);
|
||||
buf
|
||||
}
|
||||
|
||||
/// Read a single framed message from a stream.
|
||||
/// Returns (type_byte, payload). Panics on HMAC mismatch.
|
||||
async fn read_frame(stream: &mut tokio::net::TcpStream, key: &hmac::Key) -> (u8, Vec<u8>) {
|
||||
let mut header = [0u8; HEADER_SIZE];
|
||||
stream.read_exact(&mut header).await.expect("read header");
|
||||
assert_eq!(&header[..4], &PROTOCOL_MAGIC, "bad magic");
|
||||
let msg_type = header[4];
|
||||
let payload_len =
|
||||
u32::from_le_bytes([header[5], header[6], header[7], header[8]]) as usize;
|
||||
let hmac_tag = &header[9..HEADER_SIZE];
|
||||
|
||||
let mut payload = vec![0u8; payload_len];
|
||||
if payload_len > 0 {
|
||||
stream.read_exact(&mut payload).await.expect("read payload");
|
||||
}
|
||||
|
||||
// Verify HMAC
|
||||
let mut verify_data = Vec::with_capacity(9 + payload.len());
|
||||
verify_data.extend_from_slice(&header[..9]);
|
||||
verify_data.extend_from_slice(&payload);
|
||||
hmac::verify(key, &verify_data, hmac_tag)
|
||||
.expect("HMAC verification failed — wrong key or tampered data");
|
||||
|
||||
(msg_type, payload)
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn peer_hello_handshake() {
|
||||
// Simulate a sensor listening and a controller connecting
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let key = test_key();
|
||||
|
||||
let server_key = key.clone();
|
||||
let server_task = tokio::spawn(async move {
|
||||
let (mut stream, _) = listener.accept().await.unwrap();
|
||||
|
||||
// Read HELLO from client
|
||||
let (msg_type, payload) = read_frame(&mut stream, &server_key).await;
|
||||
assert_eq!(msg_type, HELLO_TYPE);
|
||||
|
||||
let hello: serde_json::Value = serde_json::from_slice(&payload).unwrap();
|
||||
assert_eq!(hello["node_id"], "test-client");
|
||||
|
||||
// Send HELLO response
|
||||
let resp = serde_json::json!({
|
||||
"node_id": "test-server",
|
||||
"version": "1.0.0",
|
||||
"blocked_count": 42
|
||||
});
|
||||
let resp_bytes = serde_json::to_vec(&resp).unwrap();
|
||||
let msg = encode_message(HELLO_TYPE, &resp_bytes, &server_key);
|
||||
stream.write_all(&msg).await.unwrap();
|
||||
stream.flush().await.unwrap();
|
||||
});
|
||||
|
||||
let client_key = key.clone();
|
||||
let client_task = tokio::spawn(async move {
|
||||
let mut stream = tokio::net::TcpStream::connect(addr).await.unwrap();
|
||||
|
||||
// Send HELLO
|
||||
let hello = serde_json::json!({
|
||||
"node_id": "test-client",
|
||||
"version": "1.0.0",
|
||||
"blocked_count": 0
|
||||
});
|
||||
let hello_bytes = serde_json::to_vec(&hello).unwrap();
|
||||
let msg = encode_message(HELLO_TYPE, &hello_bytes, &client_key);
|
||||
stream.write_all(&msg).await.unwrap();
|
||||
stream.flush().await.unwrap();
|
||||
|
||||
// Read HELLO response
|
||||
let (msg_type, payload) = read_frame(&mut stream, &client_key).await;
|
||||
assert_eq!(msg_type, HELLO_TYPE);
|
||||
|
||||
let resp: serde_json::Value = serde_json::from_slice(&payload).unwrap();
|
||||
assert_eq!(resp["node_id"], "test-server");
|
||||
assert_eq!(resp["blocked_count"], 42);
|
||||
});
|
||||
|
||||
tokio::try_join!(server_task, client_task).unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn peer_heartbeat_response_cycle() {
|
||||
// Verify that heartbeat → HELLO response cycle works over persistent TCP
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let key = test_key();
|
||||
|
||||
let server_key = key.clone();
|
||||
let server_task = tokio::spawn(async move {
|
||||
let (mut stream, _) = listener.accept().await.unwrap();
|
||||
|
||||
// Read 3 heartbeats, respond to each with HELLO
|
||||
for i in 0..3u32 {
|
||||
let (msg_type, payload) = read_frame(&mut stream, &server_key).await;
|
||||
assert_eq!(msg_type, HEARTBEAT_TYPE, "expected heartbeat {}", i);
|
||||
assert_eq!(payload.len(), 0, "heartbeat payload should be empty");
|
||||
|
||||
// Respond with HELLO containing metrics
|
||||
let resp = serde_json::json!({
|
||||
"node_id": "sensor-1",
|
||||
"version": "1.0.0",
|
||||
"blocked_count": 10 + i
|
||||
});
|
||||
let resp_bytes = serde_json::to_vec(&resp).unwrap();
|
||||
let msg = encode_message(HELLO_TYPE, &resp_bytes, &server_key);
|
||||
stream.write_all(&msg).await.unwrap();
|
||||
stream.flush().await.unwrap();
|
||||
}
|
||||
});
|
||||
|
||||
let client_key = key.clone();
|
||||
let client_task = tokio::spawn(async move {
|
||||
let mut stream = tokio::net::TcpStream::connect(addr).await.unwrap();
|
||||
|
||||
// Send 3 heartbeats on the SAME connection and verify responses
|
||||
for i in 0..3u32 {
|
||||
let heartbeat = encode_message(HEARTBEAT_TYPE, &[], &client_key);
|
||||
stream.write_all(&heartbeat).await.unwrap();
|
||||
stream.flush().await.unwrap();
|
||||
|
||||
let (msg_type, payload) = read_frame(&mut stream, &client_key).await;
|
||||
assert_eq!(msg_type, HELLO_TYPE, "expected HELLO response {}", i);
|
||||
|
||||
let resp: serde_json::Value = serde_json::from_slice(&payload).unwrap();
|
||||
assert_eq!(resp["blocked_count"], 10 + i);
|
||||
}
|
||||
// Connection still alive after 3 exchanges — persistent TCP works
|
||||
});
|
||||
|
||||
tokio::try_join!(server_task, client_task).unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn multiple_blocked_ip_on_single_connection() {
|
||||
// Verify that multiple BlockedIp messages can be sent on a single TCP stream
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let key = test_key();
|
||||
|
||||
let server_key = key.clone();
|
||||
let server_task = tokio::spawn(async move {
|
||||
let (mut stream, _) = listener.accept().await.unwrap();
|
||||
|
||||
// Read 5 blocked IPs on the same connection
|
||||
for i in 0..5u32 {
|
||||
let (msg_type, payload) = read_frame(&mut stream, &server_key).await;
|
||||
assert_eq!(msg_type, BLOCKED_IP_TYPE);
|
||||
|
||||
let blocked: serde_json::Value = serde_json::from_slice(&payload).unwrap();
|
||||
let expected_ip = format!("10.0.0.{}", i + 1);
|
||||
assert_eq!(blocked["ip"], expected_ip);
|
||||
assert!(blocked["confidence"].as_u64().unwrap() >= 50);
|
||||
}
|
||||
});
|
||||
|
||||
let client_key = key.clone();
|
||||
let client_task = tokio::spawn(async move {
|
||||
let mut stream = tokio::net::TcpStream::connect(addr).await.unwrap();
|
||||
|
||||
// Send 5 blocked IPs on the same connection
|
||||
for i in 0..5u32 {
|
||||
let payload = serde_json::json!({
|
||||
"ip": format!("10.0.0.{}", i + 1),
|
||||
"reason": "integration test",
|
||||
"duration_secs": 600,
|
||||
"confidence": 85
|
||||
});
|
||||
let payload_bytes = serde_json::to_vec(&payload).unwrap();
|
||||
let msg = encode_message(BLOCKED_IP_TYPE, &payload_bytes, &client_key);
|
||||
stream.write_all(&msg).await.unwrap();
|
||||
stream.flush().await.unwrap();
|
||||
}
|
||||
});
|
||||
|
||||
tokio::try_join!(server_task, client_task).unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn invalid_magic_rejected() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
|
||||
let server_task = tokio::spawn(async move {
|
||||
let (mut stream, _) = listener.accept().await.unwrap();
|
||||
let mut buf = [0u8; 1024];
|
||||
// Connection should be dropped by peer after sending garbage
|
||||
let result = tokio::time::timeout(
|
||||
Duration::from_secs(2),
|
||||
stream.read(&mut buf),
|
||||
).await;
|
||||
// Either timeout or read some bytes — just verify no panic
|
||||
drop(result);
|
||||
});
|
||||
|
||||
let client_task = tokio::spawn(async move {
|
||||
let mut stream = tokio::net::TcpStream::connect(addr).await.unwrap();
|
||||
// Send garbage with wrong magic (pad to HEADER_SIZE)
|
||||
let mut garbage = vec![0xFF; HEADER_SIZE];
|
||||
garbage[4] = HELLO_TYPE;
|
||||
stream.write_all(&garbage).await.unwrap();
|
||||
stream.flush().await.unwrap();
|
||||
});
|
||||
|
||||
tokio::try_join!(server_task, client_task).unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn oversized_payload_rejected() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
|
||||
let server_task = tokio::spawn(async move {
|
||||
let (mut stream, _) = listener.accept().await.unwrap();
|
||||
// Read header claiming 1MB payload — should be rejected as too large
|
||||
let mut header = [0u8; HEADER_SIZE];
|
||||
stream.read_exact(&mut header).await.unwrap();
|
||||
assert_eq!(&header[..4], &PROTOCOL_MAGIC);
|
||||
let payload_len =
|
||||
u32::from_le_bytes([header[5], header[6], header[7], header[8]]) as usize;
|
||||
assert!(payload_len > 65536, "payload should be oversized");
|
||||
// Server would reject this — test verifies the frame was parseable
|
||||
});
|
||||
|
||||
let client_task = tokio::spawn(async move {
|
||||
let mut stream = tokio::net::TcpStream::connect(addr).await.unwrap();
|
||||
// Send header with 1MB payload length (but don't send payload)
|
||||
let mut msg = vec![0u8; HEADER_SIZE];
|
||||
msg[..4].copy_from_slice(&PROTOCOL_MAGIC);
|
||||
msg[4] = HELLO_TYPE;
|
||||
let huge_len: u32 = 1_000_000;
|
||||
msg[5..9].copy_from_slice(&huge_len.to_le_bytes());
|
||||
// HMAC is garbage — but we're testing payload size rejection, not auth
|
||||
stream.write_all(&msg).await.unwrap();
|
||||
stream.flush().await.unwrap();
|
||||
});
|
||||
|
||||
tokio::try_join!(server_task, client_task).unwrap();
|
||||
}
|
||||
|
||||
/// Test that a tampered HMAC causes frame rejection.
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn hmac_tamper_detection() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let key = test_key();
|
||||
|
||||
let server_key = key.clone();
|
||||
let server_task = tokio::spawn(async move {
|
||||
let (mut stream, _) = listener.accept().await.unwrap();
|
||||
let mut header = [0u8; HEADER_SIZE];
|
||||
let result = tokio::time::timeout(
|
||||
Duration::from_secs(2),
|
||||
stream.read_exact(&mut header),
|
||||
).await;
|
||||
// The frame arrives, but HMAC verification should fail
|
||||
if let Ok(Ok(_)) = result {
|
||||
let payload_len =
|
||||
u32::from_le_bytes([header[5], header[6], header[7], header[8]]) as usize;
|
||||
let mut payload = vec![0u8; payload_len];
|
||||
if payload_len > 0 {
|
||||
let _ = stream.read_exact(&mut payload).await;
|
||||
}
|
||||
// Verify HMAC — should fail because we tampered
|
||||
let mut verify_data = Vec::with_capacity(9 + payload.len());
|
||||
verify_data.extend_from_slice(&header[..9]);
|
||||
verify_data.extend_from_slice(&payload);
|
||||
let result = hmac::verify(
|
||||
&server_key, &verify_data, &header[9..HEADER_SIZE],
|
||||
);
|
||||
assert!(result.is_err(), "tampered HMAC should fail verification");
|
||||
}
|
||||
});
|
||||
|
||||
let client_key = key.clone();
|
||||
let client_task = tokio::spawn(async move {
|
||||
let mut stream = tokio::net::TcpStream::connect(addr).await.unwrap();
|
||||
let payload = serde_json::json!({"node_id": "evil"});
|
||||
let payload_bytes = serde_json::to_vec(&payload).unwrap();
|
||||
let mut msg = encode_message(HELLO_TYPE, &payload_bytes, &client_key);
|
||||
// Tamper: flip a byte in the HMAC region (positions 9..41)
|
||||
msg[15] ^= 0xFF;
|
||||
stream.write_all(&msg).await.unwrap();
|
||||
stream.flush().await.unwrap();
|
||||
});
|
||||
|
||||
tokio::try_join!(server_task, client_task).unwrap();
|
||||
}
|
||||
|
||||
/// Test that a wrong PSK causes HMAC rejection.
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn wrong_psk_rejected() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
|
||||
let server_key = test_key();
|
||||
let wrong_key = hmac::Key::new(hmac::HMAC_SHA256, b"wrong-psk-not-matching");
|
||||
|
||||
let server_task = tokio::spawn(async move {
|
||||
let (mut stream, _) = listener.accept().await.unwrap();
|
||||
let mut header = [0u8; HEADER_SIZE];
|
||||
let result = tokio::time::timeout(
|
||||
Duration::from_secs(2),
|
||||
stream.read_exact(&mut header),
|
||||
).await;
|
||||
if let Ok(Ok(_)) = result {
|
||||
let payload_len =
|
||||
u32::from_le_bytes([header[5], header[6], header[7], header[8]]) as usize;
|
||||
let mut payload = vec![0u8; payload_len];
|
||||
if payload_len > 0 {
|
||||
let _ = stream.read_exact(&mut payload).await;
|
||||
}
|
||||
let mut verify_data = Vec::with_capacity(9 + payload.len());
|
||||
verify_data.extend_from_slice(&header[..9]);
|
||||
verify_data.extend_from_slice(&payload);
|
||||
let result = hmac::verify(
|
||||
&server_key, &verify_data, &header[9..HEADER_SIZE],
|
||||
);
|
||||
assert!(result.is_err(), "wrong PSK should fail verification");
|
||||
}
|
||||
});
|
||||
|
||||
let client_task = tokio::spawn(async move {
|
||||
let mut stream = tokio::net::TcpStream::connect(addr).await.unwrap();
|
||||
let payload = serde_json::json!({"node_id": "intruder"});
|
||||
let payload_bytes = serde_json::to_vec(&payload).unwrap();
|
||||
// Signed with wrong key
|
||||
let msg = encode_message(HELLO_TYPE, &payload_bytes, &wrong_key);
|
||||
stream.write_all(&msg).await.unwrap();
|
||||
stream.flush().await.unwrap();
|
||||
});
|
||||
|
||||
tokio::try_join!(server_task, client_task).unwrap();
|
||||
}
|
||||
17
common/Cargo.toml
Executable file
17
common/Cargo.toml
Executable file
|
|
@ -0,0 +1,17 @@
|
|||
[package]
|
||||
name = "common"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
|
||||
[features]
|
||||
default = ["user", "aya"]
|
||||
user = ["dep:serde", "dep:serde_json"]
|
||||
aya = ["dep:aya", "user"]
|
||||
|
||||
[dependencies]
|
||||
aya = { version = "0.13", optional = true }
|
||||
serde = { version = "1", features = ["derive"], optional = true }
|
||||
serde_json = { version = "1", optional = true }
|
||||
|
||||
[lib]
|
||||
path = "src/lib.rs"
|
||||
157
common/src/base64.rs
Executable file
157
common/src/base64.rs
Executable file
|
|
@ -0,0 +1,157 @@
|
|||
//! Minimal base64/base64url codec — no external crates.
|
||||
//!
|
||||
//! Used by the A2A firewall (JWT parsing, PoP verification) and
|
||||
//! the A2A shim (token encoding). A single implementation avoids
|
||||
//! drift between the three nearly-identical copies that existed before.
|
||||
|
||||
/// Standard base64 alphabet lookup table (6-bit value per ASCII char).
|
||||
const DECODE_TABLE: &[u8; 128] = &{
|
||||
let mut t = [255u8; 128];
|
||||
let mut i = 0u8;
|
||||
while i < 26 {
|
||||
t[(b'A' + i) as usize] = i;
|
||||
t[(b'a' + i) as usize] = i + 26;
|
||||
i += 1;
|
||||
}
|
||||
let mut i = 0u8;
|
||||
while i < 10 {
|
||||
t[(b'0' + i) as usize] = i + 52;
|
||||
i += 1;
|
||||
}
|
||||
t[b'+' as usize] = 62;
|
||||
t[b'/' as usize] = 63;
|
||||
t
|
||||
};
|
||||
|
||||
/// Base64url alphabet for encoding (RFC 4648 §5, no padding).
|
||||
const ENCODE_TABLE: &[u8; 64] =
|
||||
b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_";
|
||||
|
||||
/// Decode a base64url string (no padding required) into bytes.
|
||||
///
|
||||
/// Handles the URL-safe alphabet (`-` → `+`, `_` → `/`) and adds
|
||||
/// padding automatically before delegating to the standard decoder.
|
||||
pub fn decode_base64url(input: &str) -> Result<Vec<u8>, &'static str> {
|
||||
// Add padding if needed
|
||||
let padded = match input.len() % 4 {
|
||||
2 => format!("{input}=="),
|
||||
3 => format!("{input}="),
|
||||
0 => input.to_string(),
|
||||
_ => return Err("invalid base64url length"),
|
||||
};
|
||||
|
||||
// Convert base64url → standard base64
|
||||
let standard: String = padded
|
||||
.chars()
|
||||
.map(|c| match c {
|
||||
'-' => '+',
|
||||
'_' => '/',
|
||||
other => other,
|
||||
})
|
||||
.collect();
|
||||
|
||||
decode_base64_standard(&standard)
|
||||
}
|
||||
|
||||
/// Decode a standard base64 string (with `=` padding) into bytes.
|
||||
pub fn decode_base64_standard(input: &str) -> Result<Vec<u8>, &'static str> {
|
||||
let bytes = input.as_bytes();
|
||||
let len = bytes.len();
|
||||
if !len.is_multiple_of(4) {
|
||||
return Err("invalid base64 length");
|
||||
}
|
||||
|
||||
let mut out = Vec::with_capacity(len / 4 * 3);
|
||||
let mut i = 0;
|
||||
while i < len {
|
||||
let a = bytes[i];
|
||||
let b = bytes[i + 1];
|
||||
let c = bytes[i + 2];
|
||||
let d = bytes[i + 3];
|
||||
|
||||
let va = if a == b'=' { 0 } else if a > 127 { return Err("invalid char") } else { DECODE_TABLE[a as usize] };
|
||||
let vb = if b == b'=' { 0 } else if b > 127 { return Err("invalid char") } else { DECODE_TABLE[b as usize] };
|
||||
let vc = if c == b'=' { 0 } else if c > 127 { return Err("invalid char") } else { DECODE_TABLE[c as usize] };
|
||||
let vd = if d == b'=' { 0 } else if d > 127 { return Err("invalid char") } else { DECODE_TABLE[d as usize] };
|
||||
|
||||
if va == 255 || vb == 255 || vc == 255 || vd == 255 {
|
||||
return Err("invalid base64 character");
|
||||
}
|
||||
|
||||
let triple = (va as u32) << 18 | (vb as u32) << 12 | (vc as u32) << 6 | (vd as u32);
|
||||
out.push((triple >> 16) as u8);
|
||||
if c != b'=' {
|
||||
out.push((triple >> 8) as u8);
|
||||
}
|
||||
if d != b'=' {
|
||||
out.push(triple as u8);
|
||||
}
|
||||
i += 4;
|
||||
}
|
||||
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Encode bytes to base64url (no padding, RFC 4648 §5).
|
||||
pub fn encode_base64url(input: &[u8]) -> String {
|
||||
let mut out = String::with_capacity((input.len() * 4 / 3) + 4);
|
||||
for chunk in input.chunks(3) {
|
||||
let b0 = chunk[0] as u32;
|
||||
let b1 = if chunk.len() > 1 { chunk[1] as u32 } else { 0 };
|
||||
let b2 = if chunk.len() > 2 { chunk[2] as u32 } else { 0 };
|
||||
let triple = (b0 << 16) | (b1 << 8) | b2;
|
||||
|
||||
out.push(ENCODE_TABLE[((triple >> 18) & 0x3F) as usize] as char);
|
||||
out.push(ENCODE_TABLE[((triple >> 12) & 0x3F) as usize] as char);
|
||||
if chunk.len() > 1 {
|
||||
out.push(ENCODE_TABLE[((triple >> 6) & 0x3F) as usize] as char);
|
||||
}
|
||||
if chunk.len() > 2 {
|
||||
out.push(ENCODE_TABLE[(triple & 0x3F) as usize] as char);
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn round_trip() {
|
||||
let data = b"Hello, Blackwall!";
|
||||
let encoded = encode_base64url(data);
|
||||
let decoded = decode_base64url(&encoded).unwrap();
|
||||
assert_eq!(decoded, data);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn standard_base64_padding() {
|
||||
// "Man" → "TWFu"
|
||||
assert_eq!(decode_base64_standard("TWFu").unwrap(), b"Man");
|
||||
// "Ma" → "TWE="
|
||||
assert_eq!(decode_base64_standard("TWE=").unwrap(), b"Ma");
|
||||
// "M" → "TQ=="
|
||||
assert_eq!(decode_base64_standard("TQ==").unwrap(), b"M");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn url_safe_chars() {
|
||||
// '+' and '/' in standard → '-' and '_' in url-safe
|
||||
let standard = "ab+c/d==";
|
||||
let url_safe = "ab-c_d";
|
||||
let decode_std = decode_base64_standard(standard).unwrap();
|
||||
let decode_url = decode_base64url(url_safe).unwrap();
|
||||
assert_eq!(decode_std, decode_url);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_length() {
|
||||
assert!(decode_base64url("A").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_char() {
|
||||
assert!(decode_base64_standard("!!!!").is_err());
|
||||
}
|
||||
}
|
||||
407
common/src/hivemind.rs
Executable file
407
common/src/hivemind.rs
Executable file
|
|
@ -0,0 +1,407 @@
|
|||
//! HiveMind Threat Mesh — shared types for P2P threat intelligence.
|
||||
//!
|
||||
//! These types are used by both the hivemind daemon and potentially
|
||||
//! by eBPF programs that feed threat data into the mesh.
|
||||
|
||||
/// Maximum length of a JA4 fingerprint string (e.g., "t13d1516h2_8daaf6152771_e5627efa2ab1").
|
||||
pub const JA4_FINGERPRINT_LEN: usize = 36;
|
||||
|
||||
/// Maximum length of a threat description.
|
||||
pub const THREAT_DESC_LEN: usize = 128;
|
||||
|
||||
/// Maximum bootstrap nodes in configuration.
|
||||
pub const MAX_BOOTSTRAP_NODES: usize = 16;
|
||||
|
||||
/// Default GossipSub fan-out.
|
||||
pub const GOSSIPSUB_FANOUT: usize = 10;
|
||||
|
||||
/// Default GossipSub heartbeat interval in seconds.
|
||||
pub const GOSSIPSUB_HEARTBEAT_SECS: u64 = 1;
|
||||
|
||||
/// Default Kademlia query timeout in seconds.
|
||||
pub const KADEMLIA_QUERY_TIMEOUT_SECS: u64 = 60;
|
||||
|
||||
/// Maximum GossipSub message size (64 KB).
|
||||
pub const MAX_MESSAGE_SIZE: usize = 65536;
|
||||
|
||||
/// GossipSub message deduplication TTL in seconds.
|
||||
pub const MESSAGE_DEDUP_TTL_SECS: u64 = 120;
|
||||
|
||||
/// Kademlia k-bucket size.
|
||||
pub const K_BUCKET_SIZE: usize = 20;
|
||||
|
||||
/// Severity level of a threat indicator.
|
||||
#[repr(u8)]
|
||||
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
|
||||
pub enum ThreatSeverity {
|
||||
/// Informational — low confidence or low impact.
|
||||
Info = 0,
|
||||
/// Low — minor scanning or recon activity.
|
||||
Low = 1,
|
||||
/// Medium — active probing or known-bad pattern.
|
||||
Medium = 2,
|
||||
/// High — confirmed malicious with high confidence.
|
||||
High = 3,
|
||||
/// Critical — active exploitation or C2 communication.
|
||||
Critical = 4,
|
||||
}
|
||||
|
||||
impl ThreatSeverity {
|
||||
/// Convert raw u8 to ThreatSeverity.
|
||||
pub fn from_u8(v: u8) -> Self {
|
||||
match v {
|
||||
0 => ThreatSeverity::Info,
|
||||
1 => ThreatSeverity::Low,
|
||||
2 => ThreatSeverity::Medium,
|
||||
3 => ThreatSeverity::High,
|
||||
4 => ThreatSeverity::Critical,
|
||||
_ => ThreatSeverity::Info,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Type of Indicator of Compromise.
|
||||
#[repr(u8)]
|
||||
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
|
||||
pub enum IoCType {
|
||||
/// IPv4 address associated with malicious activity.
|
||||
MaliciousIp = 0,
|
||||
/// JA4 TLS fingerprint of known malware/tool.
|
||||
Ja4Fingerprint = 1,
|
||||
/// High-entropy payload pattern (encrypted C2, exfil).
|
||||
EntropyAnomaly = 2,
|
||||
/// DNS tunneling indicator.
|
||||
DnsTunnel = 3,
|
||||
/// Behavioral pattern (port scan, brute force).
|
||||
BehavioralPattern = 4,
|
||||
}
|
||||
|
||||
impl IoCType {
|
||||
/// Convert raw u8 to IoCType.
|
||||
pub fn from_u8(v: u8) -> Self {
|
||||
match v {
|
||||
0 => IoCType::MaliciousIp,
|
||||
1 => IoCType::Ja4Fingerprint,
|
||||
2 => IoCType::EntropyAnomaly,
|
||||
3 => IoCType::DnsTunnel,
|
||||
4 => IoCType::BehavioralPattern,
|
||||
_ => IoCType::MaliciousIp,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Indicator of Compromise — the core unit of threat intelligence shared
|
||||
/// across the HiveMind mesh. Designed for GossipSub transmission.
|
||||
///
|
||||
/// This is a userspace-only type (not eBPF), so we use std types freely.
|
||||
#[cfg(feature = "user")]
|
||||
#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
|
||||
pub struct IoC {
|
||||
/// Type of indicator.
|
||||
pub ioc_type: u8,
|
||||
/// Severity level.
|
||||
pub severity: u8,
|
||||
/// IPv4 address (if applicable, 0 otherwise).
|
||||
pub ip: u32,
|
||||
/// JA4 fingerprint string (if applicable).
|
||||
pub ja4: Option<String>,
|
||||
/// Byte diversity score (unique_count × 31, if applicable).
|
||||
pub entropy_score: Option<u32>,
|
||||
/// Human-readable description.
|
||||
pub description: String,
|
||||
/// Unix timestamp when this IoC was first observed.
|
||||
pub first_seen: u64,
|
||||
/// Number of independent peers that confirmed this IoC.
|
||||
pub confirmations: u32,
|
||||
/// ZKP proof blob (empty until Phase 1 implementation).
|
||||
pub zkp_proof: Vec<u8>,
|
||||
}
|
||||
|
||||
/// A threat report broadcast via GossipSub. Contains one or more IoCs
|
||||
/// from a single reporting node.
|
||||
#[cfg(feature = "user")]
|
||||
#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
|
||||
pub struct ThreatReport {
|
||||
/// Unique report ID (UUID v4 bytes).
|
||||
pub report_id: [u8; 16],
|
||||
/// Reporter's Ed25519 public key (32 bytes).
|
||||
pub reporter_pubkey: [u8; 32],
|
||||
/// Unix timestamp of the report.
|
||||
pub timestamp: u64,
|
||||
/// List of IoCs in this report.
|
||||
pub indicators: Vec<IoC>,
|
||||
/// Ed25519 signature over the serialized indicators.
|
||||
pub signature: Vec<u8>,
|
||||
}
|
||||
|
||||
/// GossipSub topic identifiers for HiveMind.
|
||||
pub mod topics {
|
||||
/// IoC broadcast topic — new threat indicators.
|
||||
pub const IOC_TOPIC: &str = "hivemind/ioc/v1";
|
||||
/// JA4 fingerprint sharing topic.
|
||||
pub const JA4_TOPIC: &str = "hivemind/ja4/v1";
|
||||
/// Federated learning gradient exchange topic.
|
||||
pub const GRADIENT_TOPIC: &str = "hivemind/federated/gradients/v1";
|
||||
/// Peer heartbeat / presence topic.
|
||||
pub const HEARTBEAT_TOPIC: &str = "hivemind/heartbeat/v1";
|
||||
/// A2A violation proof sharing topic.
|
||||
pub const A2A_VIOLATIONS_TOPIC: &str = "hivemind/a2a-violations/v1";
|
||||
}
|
||||
|
||||
/// Port for local proof ingestion IPC (enterprise module → hivemind).
|
||||
///
|
||||
/// Hivemind listens on `127.0.0.1:PROOF_INGEST_PORT` for length-prefixed
|
||||
/// proof envelopes from the enterprise daemon (optional) on the same machine.
|
||||
pub const PROOF_INGEST_PORT: u16 = 9821;
|
||||
|
||||
/// Port for local IoC injection (testing/integration).
|
||||
///
|
||||
/// Hivemind listens on `127.0.0.1:IOC_INJECT_PORT` for length-prefixed
|
||||
/// IoC JSON payloads. The injected IoC is published to GossipSub and
|
||||
/// submitted to local consensus with the node's own pubkey.
|
||||
pub const IOC_INJECT_PORT: u16 = 9822;
|
||||
|
||||
// --- Phase 1: Anti-Poisoning Constants ---
|
||||
|
||||
/// Initial reputation stake for new peers.
|
||||
///
|
||||
/// Set BELOW MIN_TRUSTED_REPUTATION so new peers must earn trust through
|
||||
/// accurate reports before participating in consensus. Prevents Sybil
|
||||
/// attacks where fresh peers immediately inject false IoCs.
|
||||
pub const INITIAL_STAKE: u64 = 30;
|
||||
|
||||
/// Minimum reputation score to be considered trusted.
|
||||
pub const MIN_TRUSTED_REPUTATION: u64 = 50;
|
||||
|
||||
/// Initial reputation stake for explicitly configured seed peers.
|
||||
/// Seed peers start trusted to bootstrap the consensus network.
|
||||
pub const SEED_PEER_STAKE: u64 = 100;
|
||||
|
||||
/// Slashing penalty for submitting a false IoC (% of stake).
|
||||
pub const SLASHING_PENALTY_PERCENT: u64 = 25;
|
||||
|
||||
/// Reward for accurate IoC report (stake units).
|
||||
pub const ACCURACY_REWARD: u64 = 5;
|
||||
|
||||
/// Minimum independent peer confirmations to accept an IoC.
|
||||
pub const CROSS_VALIDATION_THRESHOLD: usize = 3;
|
||||
|
||||
/// Time window (seconds) for pending IoC cross-validation before expiry.
|
||||
pub const CONSENSUS_TIMEOUT_SECS: u64 = 300;
|
||||
|
||||
/// Proof-of-Work difficulty for new peer registration (leading zero bits).
|
||||
pub const POW_DIFFICULTY_BITS: u32 = 20;
|
||||
|
||||
/// Maximum new peer registrations per minute (rate limit).
|
||||
pub const MAX_PEER_REGISTRATIONS_PER_MINUTE: usize = 10;
|
||||
|
||||
/// PoW challenge freshness window (seconds).
|
||||
pub const POW_CHALLENGE_TTL_SECS: u64 = 120;
|
||||
|
||||
// --- Phase 1: ZKP Stub Types ---
|
||||
|
||||
/// A zero-knowledge proof that a threat was observed without revealing
|
||||
/// raw packet data. Stub type for Phase 0-1 interface stability.
|
||||
///
|
||||
/// In future phases, this will contain a bellman/arkworks SNARK proof.
|
||||
#[cfg(feature = "user")]
|
||||
#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
|
||||
pub struct ThreatProof {
|
||||
/// Version of the proof format (for forward compatibility).
|
||||
pub version: u8,
|
||||
/// The statement being proven (what claims are made).
|
||||
pub statement: ProofStatement,
|
||||
/// Opaque proof bytes. Empty = stub, non-empty = real SNARK proof.
|
||||
pub proof_data: Vec<u8>,
|
||||
/// Unix timestamp when proof was generated.
|
||||
pub created_at: u64,
|
||||
}
|
||||
|
||||
/// What a ZKP proof claims to demonstrate, without revealing private inputs.
|
||||
#[cfg(feature = "user")]
|
||||
#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
|
||||
pub struct ProofStatement {
|
||||
/// JA4 fingerprint hash that was matched (public output).
|
||||
pub ja4_hash: Option<[u8; 32]>,
|
||||
/// Whether entropy exceeded the anomaly threshold (public output).
|
||||
pub entropy_exceeded: bool,
|
||||
/// Whether the behavioral classifier labeled this as malicious.
|
||||
pub classified_malicious: bool,
|
||||
/// IoC type this proof covers.
|
||||
pub ioc_type: u8,
|
||||
}
|
||||
|
||||
/// Proof-of-Work challenge for Sybil resistance.
|
||||
#[cfg(feature = "user")]
|
||||
#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
|
||||
pub struct PowChallenge {
|
||||
/// The peer's public key (32 bytes Ed25519).
|
||||
pub peer_pubkey: [u8; 32],
|
||||
/// Nonce found by the peer that satisfies difficulty.
|
||||
pub nonce: u64,
|
||||
/// Timestamp when the PoW was computed.
|
||||
pub timestamp: u64,
|
||||
/// Difficulty in leading zero bits.
|
||||
pub difficulty: u32,
|
||||
}
|
||||
|
||||
/// Peer reputation record shared across the mesh.
|
||||
#[cfg(feature = "user")]
|
||||
#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
|
||||
pub struct PeerReputationRecord {
|
||||
/// Ed25519 public key of the peer (32 bytes).
|
||||
pub peer_pubkey: [u8; 32],
|
||||
/// Current stake (starts at INITIAL_STAKE).
|
||||
pub stake: u64,
|
||||
/// Cumulative accuracy score (accurate reports).
|
||||
pub accurate_reports: u64,
|
||||
/// Count of false positives flagged by consensus.
|
||||
pub false_reports: u64,
|
||||
/// Unix timestamp of last activity.
|
||||
pub last_active: u64,
|
||||
}
|
||||
|
||||
// --- Phase 2: Federated Learning Constants ---
|
||||
|
||||
/// Interval between federated learning aggregation rounds (seconds).
|
||||
pub const FL_ROUND_INTERVAL_SECS: u64 = 60;
|
||||
|
||||
/// Minimum peers required to run an aggregation round.
|
||||
pub const FL_MIN_PEERS_PER_ROUND: usize = 3;
|
||||
|
||||
/// Maximum serialized gradient payload size (bytes). 16 KB.
|
||||
pub const FL_MAX_GRADIENT_SIZE: usize = 16384;
|
||||
|
||||
/// Percentage of extreme values to trim in Byzantine-resistant FedAvg.
|
||||
/// Trims top and bottom 20% of gradient contributions per dimension.
|
||||
pub const FL_BYZANTINE_TRIM_PERCENT: usize = 20;
|
||||
|
||||
/// Feature vector dimension for the local NIDS model.
|
||||
pub const FL_FEATURE_DIM: usize = 32;
|
||||
|
||||
/// Hidden layer size for the local NIDS model.
|
||||
pub const FL_HIDDEN_DIM: usize = 16;
|
||||
|
||||
/// Z-score threshold × 1000 for gradient anomaly detection.
|
||||
/// Value of 3000 means z-score > 3.0 triggers alarm.
|
||||
pub const GRADIENT_ANOMALY_ZSCORE_THRESHOLD: u64 = 3000;
|
||||
|
||||
/// Maximum gradient norm (squared, integer) before rejection.
|
||||
/// Prevents gradient explosion attacks.
|
||||
pub const FL_MAX_GRADIENT_NORM_SQ: u64 = 1_000_000;
|
||||
|
||||
// --- Phase 2: Federated Learning Types ---
|
||||
|
||||
/// Encrypted gradient update broadcast via GossipSub.
|
||||
///
|
||||
/// Privacy invariant: raw gradients NEVER leave the node.
|
||||
/// Only FHE-encrypted ciphertext is transmitted.
|
||||
#[cfg(feature = "user")]
|
||||
#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
|
||||
pub struct GradientUpdate {
|
||||
/// Reporter's Ed25519 public key (32 bytes).
|
||||
pub peer_pubkey: [u8; 32],
|
||||
/// Aggregation round identifier.
|
||||
pub round_id: u64,
|
||||
/// FHE-encrypted gradient payload. Raw gradients NEVER transmitted.
|
||||
pub encrypted_gradients: Vec<u8>,
|
||||
/// Unix timestamp of gradient computation.
|
||||
pub timestamp: u64,
|
||||
}
|
||||
|
||||
/// Result of a federated aggregation round.
|
||||
#[cfg(feature = "user")]
|
||||
#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
|
||||
pub struct AggregatedModel {
|
||||
/// Aggregation round identifier.
|
||||
pub round_id: u64,
|
||||
/// Aggregated model weights (after FedAvg).
|
||||
pub weights: Vec<f32>,
|
||||
/// Number of peers that contributed to this round.
|
||||
pub participant_count: usize,
|
||||
/// Unix timestamp of aggregation completion.
|
||||
pub timestamp: u64,
|
||||
}
|
||||
|
||||
// --- Phase 3: Enterprise Threat Feed Constants ---
|
||||
|
||||
/// Default HTTP port for the Enterprise Threat Feed API.
|
||||
///
|
||||
/// Uses 8090 (not 8443) because all traffic is plaintext HTTP on loopback.
|
||||
/// Port 8443 conventionally implies HTTPS and would be misleading.
|
||||
pub const API_DEFAULT_PORT: u16 = 8090;
|
||||
|
||||
/// Default API listen address.
|
||||
pub const API_DEFAULT_ADDR: &str = "127.0.0.1";
|
||||
|
||||
/// Maximum IoCs returned per API request.
|
||||
pub const API_MAX_PAGE_SIZE: usize = 1000;
|
||||
|
||||
/// Default IoCs per page in API responses.
|
||||
pub const API_DEFAULT_PAGE_SIZE: usize = 100;
|
||||
|
||||
/// API key length in bytes (hex-encoded = 64 chars).
|
||||
pub const API_KEY_LENGTH: usize = 32;
|
||||
|
||||
/// TAXII 2.1 content type header value.
|
||||
pub const TAXII_CONTENT_TYPE: &str = "application/taxii+json;version=2.1";
|
||||
|
||||
/// STIX 2.1 content type header value.
|
||||
pub const STIX_CONTENT_TYPE: &str = "application/stix+json;version=2.1";
|
||||
|
||||
/// TAXII collection ID for the primary threat feed.
|
||||
pub const TAXII_COLLECTION_ID: &str = "hivemind-threat-feed-v1";
|
||||
|
||||
/// TAXII collection title.
|
||||
pub const TAXII_COLLECTION_TITLE: &str = "HiveMind Verified Threat Feed";
|
||||
|
||||
/// STIX spec version.
|
||||
pub const STIX_SPEC_VERSION: &str = "2.1";
|
||||
|
||||
/// Product vendor name for SIEM formats.
|
||||
pub const SIEM_VENDOR: &str = "Blackwall";
|
||||
|
||||
/// Product name for SIEM formats.
|
||||
pub const SIEM_PRODUCT: &str = "HiveMind";
|
||||
|
||||
/// Product version for SIEM formats.
|
||||
pub const SIEM_VERSION: &str = "1.0";
|
||||
|
||||
/// Splunk sourcetype for HiveMind events.
|
||||
pub const SPLUNK_SOURCETYPE: &str = "hivemind:threat_feed";
|
||||
|
||||
// --- Phase 3: Enterprise API Tier Types ---
|
||||
|
||||
/// API access tier determining rate limits and format availability.
|
||||
#[cfg(feature = "user")]
|
||||
#[derive(Copy, Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
pub enum ApiTier {
|
||||
/// Free tier: JSON feed, limited page size.
|
||||
Free,
|
||||
/// Enterprise tier: all formats, full page size, STIX/TAXII.
|
||||
Enterprise,
|
||||
/// National security tier: full access + macro-analytics.
|
||||
NationalSecurity,
|
||||
}
|
||||
|
||||
#[cfg(feature = "user")]
|
||||
impl ApiTier {
|
||||
/// Maximum page size allowed for this tier.
|
||||
pub fn max_page_size(self) -> usize {
|
||||
match self {
|
||||
ApiTier::Free => 50,
|
||||
ApiTier::Enterprise => API_MAX_PAGE_SIZE,
|
||||
ApiTier::NationalSecurity => API_MAX_PAGE_SIZE,
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether this tier can access SIEM integration formats.
|
||||
pub fn can_access_siem(self) -> bool {
|
||||
matches!(self, ApiTier::Enterprise | ApiTier::NationalSecurity)
|
||||
}
|
||||
|
||||
/// Whether this tier can access STIX/TAXII endpoints.
|
||||
pub fn can_access_taxii(self) -> bool {
|
||||
matches!(self, ApiTier::Enterprise | ApiTier::NationalSecurity)
|
||||
}
|
||||
}
|
||||
562
common/src/lib.rs
Executable file
562
common/src/lib.rs
Executable file
|
|
@ -0,0 +1,562 @@
|
|||
#![cfg_attr(not(feature = "user"), no_std)]
|
||||
|
||||
#[cfg(feature = "user")]
|
||||
pub mod base64;
|
||||
pub mod hivemind;
|
||||
|
||||
/// Action to take on a matched rule.
|
||||
#[repr(u8)]
|
||||
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
|
||||
pub enum RuleAction {
|
||||
/// Allow packet through
|
||||
Pass = 0,
|
||||
/// Drop packet silently
|
||||
Drop = 1,
|
||||
/// Redirect to tarpit honeypot
|
||||
RedirectTarpit = 2,
|
||||
}
|
||||
|
||||
/// Packet event emitted from eBPF via RingBuf when anomaly detected.
|
||||
/// 32 bytes, naturally aligned, zero-copy safe.
|
||||
#[repr(C)]
|
||||
#[derive(Copy, Clone)]
|
||||
pub struct PacketEvent {
|
||||
/// Source IPv4 address (network byte order)
|
||||
pub src_ip: u32,
|
||||
/// Destination IPv4 address (network byte order)
|
||||
pub dst_ip: u32,
|
||||
/// Source port (network byte order)
|
||||
pub src_port: u16,
|
||||
/// Destination port (network byte order)
|
||||
pub dst_port: u16,
|
||||
/// IP protocol number (6=TCP, 17=UDP, 1=ICMP)
|
||||
pub protocol: u8,
|
||||
/// TCP flags bitmask (SYN=0x02, ACK=0x10, RST=0x04, FIN=0x01)
|
||||
pub flags: u8,
|
||||
/// Number of payload bytes analyzed for entropy
|
||||
pub payload_len: u16,
|
||||
/// Byte diversity score: unique_count × 31 (range 0–7936).
|
||||
/// NOT Shannon entropy — uses bitmap popcount heuristic in eBPF.
|
||||
pub entropy_score: u32,
|
||||
/// Lower 32 bits of bpf_ktime_get_ns()
|
||||
pub timestamp_ns: u32,
|
||||
/// Reserved padding for alignment
|
||||
pub _padding: u32,
|
||||
/// Total IP packet size in bytes
|
||||
pub packet_size: u32,
|
||||
}
|
||||
|
||||
/// Key for IP blocklist/allowlist HashMap.
|
||||
#[repr(C)]
|
||||
#[derive(Copy, Clone)]
|
||||
pub struct RuleKey {
|
||||
pub ip: u32,
|
||||
}
|
||||
|
||||
/// Value for IP blocklist/allowlist HashMap.
|
||||
#[repr(C)]
|
||||
#[derive(Copy, Clone)]
|
||||
pub struct RuleValue {
|
||||
/// Action: 0=Pass, 1=Drop, 2=RedirectTarpit
|
||||
pub action: u8,
|
||||
pub _pad1: u8,
|
||||
pub _pad2: u16,
|
||||
/// Expiry in seconds since boot (0 = permanent)
|
||||
pub expires_at: u32,
|
||||
}
|
||||
|
||||
/// Key for LpmTrie CIDR matching.
|
||||
#[repr(C)]
|
||||
#[derive(Copy, Clone)]
|
||||
pub struct CidrKey {
|
||||
/// Prefix length (0-32)
|
||||
pub prefix_len: u32,
|
||||
/// Network address (network byte order)
|
||||
pub ip: u32,
|
||||
}
|
||||
|
||||
/// Global statistics counters.
|
||||
#[repr(C)]
|
||||
#[derive(Copy, Clone)]
|
||||
pub struct Counters {
|
||||
pub packets_total: u64,
|
||||
pub packets_passed: u64,
|
||||
pub packets_dropped: u64,
|
||||
pub anomalies_sent: u64,
|
||||
}
|
||||
|
||||
/// Maximum cipher suite IDs to capture from TLS ClientHello.
|
||||
pub const TLS_MAX_CIPHERS: usize = 20;
|
||||
|
||||
/// Maximum extension IDs to capture from TLS ClientHello.
|
||||
pub const TLS_MAX_EXTENSIONS: usize = 20;
|
||||
|
||||
/// Maximum SNI hostname bytes to capture.
|
||||
pub const TLS_MAX_SNI: usize = 32;
|
||||
|
||||
/// TLS ClientHello raw components emitted from eBPF for JA4 assembly.
|
||||
/// Contains the raw fields needed to compute JA4 fingerprint in userspace.
|
||||
/// 128 bytes total, naturally aligned.
|
||||
#[repr(C)]
|
||||
#[derive(Copy, Clone)]
|
||||
pub struct TlsComponentsEvent {
|
||||
/// Source IPv4 address (network byte order on LE host)
|
||||
pub src_ip: u32,
|
||||
/// Destination IPv4 address
|
||||
pub dst_ip: u32,
|
||||
/// Source port (host byte order)
|
||||
pub src_port: u16,
|
||||
/// Destination port (host byte order)
|
||||
pub dst_port: u16,
|
||||
/// TLS version from ClientHello (e.g., 0x0303 = TLS 1.2)
|
||||
pub tls_version: u16,
|
||||
/// Number of cipher suites in ClientHello
|
||||
pub cipher_count: u8,
|
||||
/// Number of extensions in ClientHello
|
||||
pub ext_count: u8,
|
||||
/// First N cipher suite IDs (network byte order)
|
||||
pub ciphers: [u16; TLS_MAX_CIPHERS],
|
||||
/// First N extension type IDs (network byte order)
|
||||
pub extensions: [u16; TLS_MAX_EXTENSIONS],
|
||||
/// SNI hostname (first 32 bytes, null-padded)
|
||||
pub sni: [u8; TLS_MAX_SNI],
|
||||
/// ALPN first protocol length (0 if no ALPN)
|
||||
pub alpn_first_len: u8,
|
||||
/// Whether SNI extension was present
|
||||
pub has_sni: u8,
|
||||
/// Lower 32 bits of bpf_ktime_get_ns()
|
||||
pub timestamp_ns: u32,
|
||||
/// Padding to 140 bytes
|
||||
pub _padding: [u8; 2],
|
||||
}
|
||||
|
||||
/// Egress event emitted from TC classifier for outbound traffic analysis.
|
||||
/// 32 bytes, naturally aligned, zero-copy safe.
|
||||
#[repr(C)]
|
||||
#[derive(Copy, Clone)]
|
||||
pub struct EgressEvent {
|
||||
/// Source IPv4 address (local server)
|
||||
pub src_ip: u32,
|
||||
/// Destination IPv4 address (remote)
|
||||
pub dst_ip: u32,
|
||||
/// Source port
|
||||
pub src_port: u16,
|
||||
/// Destination port
|
||||
pub dst_port: u16,
|
||||
/// IP protocol (6=TCP, 17=UDP)
|
||||
pub protocol: u8,
|
||||
/// TCP flags (if TCP)
|
||||
pub flags: u8,
|
||||
/// Payload length in bytes
|
||||
pub payload_len: u16,
|
||||
/// DNS query name length (0 if not DNS)
|
||||
pub dns_query_len: u16,
|
||||
/// Entropy score of outbound payload (same scale as ingress)
|
||||
pub entropy_score: u16,
|
||||
/// Lower 32 bits of bpf_ktime_get_ns()
|
||||
pub timestamp_ns: u32,
|
||||
/// Total packet size
|
||||
pub packet_size: u32,
|
||||
}
|
||||
|
||||
/// Detected protocol from DPI tail call analysis.
|
||||
#[repr(u8)]
|
||||
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
|
||||
pub enum DpiProtocol {
|
||||
/// Unknown protocol
|
||||
Unknown = 0,
|
||||
/// HTTP (detected by method keyword)
|
||||
Http = 1,
|
||||
/// SSH (detected by "SSH-" banner)
|
||||
Ssh = 2,
|
||||
/// DNS (detected by port 53 + valid structure)
|
||||
Dns = 3,
|
||||
/// TLS (handled separately via TlsComponentsEvent)
|
||||
Tls = 4,
|
||||
}
|
||||
|
||||
impl DpiProtocol {
|
||||
/// Convert a raw u8 value to DpiProtocol.
|
||||
pub fn from_u8(v: u8) -> Self {
|
||||
match v {
|
||||
1 => DpiProtocol::Http,
|
||||
2 => DpiProtocol::Ssh,
|
||||
3 => DpiProtocol::Dns,
|
||||
4 => DpiProtocol::Tls,
|
||||
_ => DpiProtocol::Unknown,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// DPI event emitted from eBPF tail call programs via RingBuf.
|
||||
/// 24 bytes, naturally aligned, zero-copy safe.
|
||||
#[repr(C)]
|
||||
#[derive(Copy, Clone)]
|
||||
pub struct DpiEvent {
|
||||
/// Source IPv4 address
|
||||
pub src_ip: u32,
|
||||
/// Destination IPv4 address
|
||||
pub dst_ip: u32,
|
||||
/// Source port
|
||||
pub src_port: u16,
|
||||
/// Destination port
|
||||
pub dst_port: u16,
|
||||
/// Detected protocol (DpiProtocol as u8)
|
||||
pub protocol: u8,
|
||||
/// Protocol-specific flags (e.g., suspicious path for HTTP, tunneling for DNS)
|
||||
pub flags: u8,
|
||||
/// Payload length
|
||||
pub payload_len: u16,
|
||||
/// Lower 32 bits of bpf_ktime_get_ns()
|
||||
pub timestamp_ns: u32,
|
||||
}
|
||||
|
||||
/// DPI flags for HTTP detection.
|
||||
pub const DPI_HTTP_FLAG_SUSPICIOUS_PATH: u8 = 0x01;
|
||||
/// DPI flags for DNS detection.
|
||||
pub const DPI_DNS_FLAG_LONG_QUERY: u8 = 0x01;
|
||||
pub const DPI_DNS_FLAG_TUNNELING_SUSPECT: u8 = 0x02;
|
||||
/// DPI flags for SSH detection.
|
||||
pub const DPI_SSH_FLAG_SUSPICIOUS_SW: u8 = 0x01;
|
||||
|
||||
/// RingBuf size for DPI events (64 KB, power of 2).
|
||||
pub const DPI_RINGBUF_SIZE_BYTES: u32 = 64 * 1024;
|
||||
|
||||
// --- eBPF Native DNAT Types ---
|
||||
|
||||
/// Tarpit DNAT configuration pushed from userspace into eBPF map.
|
||||
/// PerCpuArray[0] — single-element scratch for tarpit routing.
|
||||
/// 16 bytes, naturally aligned, zero-copy safe.
|
||||
#[repr(C)]
|
||||
#[derive(Copy, Clone)]
|
||||
pub struct TarpitTarget {
|
||||
/// Tarpit listen port (host byte order).
|
||||
pub port: u16,
|
||||
/// Padding for alignment.
|
||||
pub _pad: u16,
|
||||
/// Local interface IP (network byte order as stored by eBPF).
|
||||
/// Used for response matching in TC reverse-NAT.
|
||||
pub local_ip: u32,
|
||||
/// Whether DNAT is enabled (1=yes, 0=no).
|
||||
pub enabled: u32,
|
||||
/// Reserved for future use.
|
||||
pub _reserved: u32,
|
||||
}
|
||||
|
||||
/// NAT tracking key for tarpit DNAT connections.
|
||||
/// Identifies a unique inbound flow from an attacker.
|
||||
/// 8 bytes, naturally aligned.
|
||||
#[repr(C)]
|
||||
#[derive(Copy, Clone)]
|
||||
pub struct NatKey {
|
||||
/// Attacker source IP (network byte order).
|
||||
pub src_ip: u32,
|
||||
/// Attacker source port (host byte order, stored as u32 for BPF alignment).
|
||||
pub src_port: u32,
|
||||
}
|
||||
|
||||
/// NAT tracking value storing the original destination before DNAT rewrite.
|
||||
/// 8 bytes, naturally aligned.
|
||||
#[repr(C)]
|
||||
#[derive(Copy, Clone)]
|
||||
pub struct NatValue {
|
||||
/// Original destination port before DNAT (host byte order).
|
||||
pub orig_dst_port: u16,
|
||||
pub _pad: u16,
|
||||
/// Timestamp (lower 32 bits of bpf_ktime_get_ns / 1e9 for LRU approx).
|
||||
pub timestamp: u32,
|
||||
}
|
||||
|
||||
/// Connection tracking key — 5-tuple identifying a unique flow.
|
||||
/// 16 bytes, naturally aligned.
|
||||
#[repr(C)]
|
||||
#[derive(Copy, Clone)]
|
||||
pub struct ConnTrackKey {
|
||||
/// Source IP (network byte order).
|
||||
pub src_ip: u32,
|
||||
/// Destination IP (network byte order).
|
||||
pub dst_ip: u32,
|
||||
/// Source port (host byte order).
|
||||
pub src_port: u16,
|
||||
/// Destination port (host byte order).
|
||||
pub dst_port: u16,
|
||||
/// IP protocol (6=TCP, 17=UDP).
|
||||
pub protocol: u8,
|
||||
pub _pad: [u8; 3],
|
||||
}
|
||||
|
||||
/// Connection tracking value — per-flow state and counters.
|
||||
/// 16 bytes, naturally aligned.
|
||||
#[repr(C)]
|
||||
#[derive(Copy, Clone)]
|
||||
pub struct ConnTrackValue {
|
||||
/// TCP state: 0=NEW, 1=SYN_SENT, 2=SYN_RECV, 3=ESTABLISHED,
|
||||
/// 4=FIN_WAIT, 5=CLOSE_WAIT, 6=CLOSED
|
||||
pub state: u8,
|
||||
/// Cumulative TCP flags seen in this flow.
|
||||
pub flags_seen: u8,
|
||||
pub _pad: u16,
|
||||
/// Packet count in this flow.
|
||||
pub packet_count: u32,
|
||||
/// Total bytes transferred.
|
||||
pub byte_count: u32,
|
||||
/// Timestamp of last packet (lower 32 of bpf_ktime_get_ns).
|
||||
pub last_seen: u32,
|
||||
}
|
||||
|
||||
/// Per-IP rate limit token bucket state for XDP rate limiting.
|
||||
/// 8 bytes, naturally aligned.
|
||||
#[repr(C)]
|
||||
#[derive(Copy, Clone)]
|
||||
pub struct RateLimitValue {
|
||||
/// Available tokens (decremented per packet, refilled per second).
|
||||
pub tokens: u32,
|
||||
/// Last refill timestamp (seconds since boot, from bpf_ktime_get_boot_ns).
|
||||
pub last_refill: u32,
|
||||
}
|
||||
|
||||
/// TCP connection states for ConnTrackValue.state
|
||||
pub const CT_STATE_NEW: u8 = 0;
|
||||
pub const CT_STATE_SYN_SENT: u8 = 1;
|
||||
pub const CT_STATE_SYN_RECV: u8 = 2;
|
||||
pub const CT_STATE_ESTABLISHED: u8 = 3;
|
||||
pub const CT_STATE_FIN_WAIT: u8 = 4;
|
||||
pub const CT_STATE_CLOSE_WAIT: u8 = 5;
|
||||
pub const CT_STATE_CLOSED: u8 = 6;
|
||||
|
||||
/// PROG_ARRAY indices for DPI tail call programs.
|
||||
pub const DPI_PROG_HTTP: u32 = 0;
|
||||
pub const DPI_PROG_DNS: u32 = 1;
|
||||
pub const DPI_PROG_SSH: u32 = 2;
|
||||
|
||||
// --- Pod safety (aya requirement for BPF map types, userspace only) ---
|
||||
// SAFETY: All types are #[repr(C)], contain only fixed-width integers,
|
||||
// have no padding holes (explicit padding fields), and no pointers.
|
||||
// eBPF side has no Pod trait — types just need #[repr(C)] + Copy.
|
||||
|
||||
#[cfg(feature = "aya")]
|
||||
unsafe impl aya::Pod for PacketEvent {}
|
||||
#[cfg(feature = "aya")]
|
||||
unsafe impl aya::Pod for RuleKey {}
|
||||
#[cfg(feature = "aya")]
|
||||
unsafe impl aya::Pod for RuleValue {}
|
||||
#[cfg(feature = "aya")]
|
||||
unsafe impl aya::Pod for CidrKey {}
|
||||
#[cfg(feature = "aya")]
|
||||
unsafe impl aya::Pod for Counters {}
|
||||
#[cfg(feature = "aya")]
|
||||
unsafe impl aya::Pod for TlsComponentsEvent {}
|
||||
#[cfg(feature = "aya")]
|
||||
unsafe impl aya::Pod for EgressEvent {}
|
||||
#[cfg(feature = "aya")]
|
||||
unsafe impl aya::Pod for DpiEvent {}
|
||||
#[cfg(feature = "aya")]
|
||||
unsafe impl aya::Pod for TarpitTarget {}
|
||||
#[cfg(feature = "aya")]
|
||||
unsafe impl aya::Pod for NatKey {}
|
||||
#[cfg(feature = "aya")]
|
||||
unsafe impl aya::Pod for NatValue {}
|
||||
#[cfg(feature = "aya")]
|
||||
unsafe impl aya::Pod for ConnTrackKey {}
|
||||
#[cfg(feature = "aya")]
|
||||
unsafe impl aya::Pod for ConnTrackValue {}
|
||||
#[cfg(feature = "aya")]
|
||||
unsafe impl aya::Pod for RateLimitValue {}
|
||||
|
||||
// --- Constants ---
|
||||
|
||||
/// TLS record content type for Handshake.
|
||||
pub const TLS_CONTENT_TYPE_HANDSHAKE: u8 = 22;
|
||||
|
||||
/// TLS handshake type for ClientHello.
|
||||
pub const TLS_HANDSHAKE_CLIENT_HELLO: u8 = 1;
|
||||
|
||||
/// RingBuf size for TLS events (64 KB, power of 2).
|
||||
pub const TLS_RINGBUF_SIZE_BYTES: u32 = 64 * 1024;
|
||||
|
||||
/// RingBuf size for egress events (64 KB, power of 2).
|
||||
pub const EGRESS_RINGBUF_SIZE_BYTES: u32 = 64 * 1024;
|
||||
|
||||
/// DNS query name length threshold for tunneling detection.
|
||||
pub const DNS_TUNNEL_QUERY_LEN_THRESHOLD: u16 = 200;
|
||||
|
||||
/// Byte diversity threshold. Payloads above this → anomaly event.
|
||||
/// Scale: unique_count × 31 (encrypted traffic: ~7000–7936, ASCII: ~1200–1800).
|
||||
pub const ENTROPY_ANOMALY_THRESHOLD: u32 = 6500;
|
||||
|
||||
/// Maximum payload bytes to analyze for entropy (must fit in eBPF bounded loop).
|
||||
pub const MAX_PAYLOAD_ANALYSIS_BYTES: usize = 128;
|
||||
|
||||
/// RingBuf size in bytes (must be power of 2). 256 KB.
|
||||
pub const RINGBUF_SIZE_BYTES: u32 = 256 * 1024;
|
||||
|
||||
/// Maximum entries in IP blocklist HashMap.
|
||||
pub const BLOCKLIST_MAX_ENTRIES: u32 = 65536;
|
||||
|
||||
/// Maximum entries in CIDR LpmTrie.
|
||||
pub const CIDR_MAX_ENTRIES: u32 = 4096;
|
||||
|
||||
/// Maximum entries in NAT tracking table (per-connection tarpit DNAT).
|
||||
pub const NAT_TABLE_MAX_ENTRIES: u32 = 65536;
|
||||
|
||||
/// Maximum entries in connection tracking LRU map.
|
||||
pub const CONN_TRACK_MAX_ENTRIES: u32 = 131072;
|
||||
|
||||
/// Maximum entries in per-IP rate limit LRU map.
|
||||
pub const RATE_LIMIT_MAX_ENTRIES: u32 = 131072;
|
||||
|
||||
/// Rate limit: max packets per second per IP before XDP_DROP.
|
||||
pub const RATE_LIMIT_PPS: u32 = 100;
|
||||
|
||||
/// Rate limit: burst capacity (tokens). Allows short bursts above PPS.
|
||||
pub const RATE_LIMIT_BURST: u32 = 200;
|
||||
|
||||
/// Tarpit default port.
|
||||
pub const TARPIT_PORT: u16 = 2222;
|
||||
|
||||
/// Tarpit base delay milliseconds.
|
||||
pub const TARPIT_BASE_DELAY_MS: u64 = 50;
|
||||
|
||||
/// Tarpit max delay milliseconds.
|
||||
pub const TARPIT_MAX_DELAY_MS: u64 = 500;
|
||||
|
||||
/// Tarpit jitter range milliseconds.
|
||||
pub const TARPIT_JITTER_MS: u64 = 100;
|
||||
|
||||
/// Tarpit min chunk size (bytes).
|
||||
pub const TARPIT_MIN_CHUNK: usize = 1;
|
||||
|
||||
/// Tarpit max chunk size (bytes).
|
||||
pub const TARPIT_MAX_CHUNK: usize = 15;
|
||||
|
||||
// --- Helper functions (std-only) ---
|
||||
|
||||
#[cfg(feature = "user")]
|
||||
pub mod util {
|
||||
use core::net::Ipv4Addr;
|
||||
|
||||
/// Convert u32 (network byte order stored on LE host) to displayable IPv4.
|
||||
///
|
||||
/// eBPF reads IP header fields as raw u32 on bpfel (little-endian).
|
||||
/// The wire bytes [A,B,C,D] become a LE u32 value. `u32::from_be()`
|
||||
/// converts that to a host-order value that `Ipv4Addr::from(u32)` expects.
|
||||
pub fn ip_from_u32(ip: u32) -> Ipv4Addr {
|
||||
Ipv4Addr::from(u32::from_be(ip))
|
||||
}
|
||||
|
||||
/// Convert IPv4 to u32 matching eBPF's bpfel representation.
|
||||
///
|
||||
/// `Ipv4Addr → u32` yields a host-order value (MSB = first octet).
|
||||
/// `.to_be()` converts to the same representation eBPF stores.
|
||||
pub fn ip_to_u32(ip: Ipv4Addr) -> u32 {
|
||||
u32::from(ip).to_be()
|
||||
}
|
||||
}
|
||||
|
||||
// --- Tests ---
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use core::mem;
|
||||
|
||||
#[test]
|
||||
fn packet_event_size_and_alignment() {
|
||||
assert_eq!(mem::size_of::<PacketEvent>(), 32);
|
||||
assert_eq!(mem::align_of::<PacketEvent>(), 4);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rule_key_size() {
|
||||
assert_eq!(mem::size_of::<RuleKey>(), 4);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rule_value_size() {
|
||||
assert_eq!(mem::size_of::<RuleValue>(), 8);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cidr_key_size() {
|
||||
assert_eq!(mem::size_of::<CidrKey>(), 8);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn counters_size() {
|
||||
assert_eq!(mem::size_of::<Counters>(), 32);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tls_components_event_size() {
|
||||
assert_eq!(mem::size_of::<TlsComponentsEvent>(), 140);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tls_components_event_alignment() {
|
||||
assert_eq!(mem::align_of::<TlsComponentsEvent>(), 4);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn egress_event_size() {
|
||||
assert_eq!(mem::size_of::<EgressEvent>(), 28);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn egress_event_alignment() {
|
||||
assert_eq!(mem::align_of::<EgressEvent>(), 4);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn entropy_threshold_in_range() {
|
||||
assert!(ENTROPY_ANOMALY_THRESHOLD <= 8000);
|
||||
assert!(ENTROPY_ANOMALY_THRESHOLD > 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ringbuf_size_is_power_of_two() {
|
||||
assert!(RINGBUF_SIZE_BYTES.is_power_of_two());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ip_conversion_roundtrip() {
|
||||
use util::*;
|
||||
let ip = core::net::Ipv4Addr::new(192, 168, 1, 1);
|
||||
let raw = ip_to_u32(ip);
|
||||
assert_eq!(ip_from_u32(raw), ip);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dpi_event_size() {
|
||||
assert_eq!(mem::size_of::<DpiEvent>(), 20);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dpi_event_alignment() {
|
||||
assert_eq!(mem::align_of::<DpiEvent>(), 4);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tarpit_target_size() {
|
||||
assert_eq!(mem::size_of::<TarpitTarget>(), 16);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nat_key_size() {
|
||||
assert_eq!(mem::size_of::<NatKey>(), 8);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nat_value_size() {
|
||||
assert_eq!(mem::size_of::<NatValue>(), 8);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn conn_track_key_size() {
|
||||
assert_eq!(mem::size_of::<ConnTrackKey>(), 16);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn conn_track_value_size() {
|
||||
assert_eq!(mem::size_of::<ConnTrackValue>(), 16);
|
||||
}
|
||||
}
|
||||
43
config.toml.example
Executable file
43
config.toml.example
Executable file
|
|
@ -0,0 +1,43 @@
|
|||
# Blackwall — Example Configuration
|
||||
# Copy to config.toml and adjust for your environment.
|
||||
|
||||
[network]
|
||||
# Network interface to attach XDP program to
|
||||
interface = "eth0"
|
||||
# XDP attach mode: "generic", "native", or "offload"
|
||||
xdp_mode = "generic"
|
||||
|
||||
[thresholds]
|
||||
# Byte diversity score above which a packet is anomalous (range 0–7936)
|
||||
entropy_anomaly = 6000
|
||||
|
||||
[tarpit]
|
||||
enabled = true
|
||||
# Port the tarpit honeypot listens on
|
||||
port = 2222
|
||||
# Jitter parameters (milliseconds)
|
||||
base_delay_ms = 100
|
||||
max_delay_ms = 30000
|
||||
jitter_ms = 500
|
||||
|
||||
[ai]
|
||||
enabled = true
|
||||
# Ollama API endpoint
|
||||
ollama_url = "http://localhost:11434"
|
||||
# Primary and fallback LLM models (must be ≤3B params for 8GB VRAM)
|
||||
model = "qwen3:1.7b"
|
||||
fallback_model = "qwen3:0.6b"
|
||||
# Max tokens for classification response
|
||||
max_tokens = 512
|
||||
# Timeout for LLM requests (milliseconds)
|
||||
timeout_ms = 5000
|
||||
|
||||
[rules]
|
||||
# Static blocklist — IPs to always DROP
|
||||
blocklist = [
|
||||
# "192.168.1.100",
|
||||
]
|
||||
# Static allowlist — IPs to always PASS
|
||||
allowlist = [
|
||||
"127.0.0.1",
|
||||
]
|
||||
20
deploy/docker/Dockerfile.blackwall
Executable file
20
deploy/docker/Dockerfile.blackwall
Executable file
|
|
@ -0,0 +1,20 @@
|
|||
# Blackwall userspace daemon — multi-stage build
|
||||
# Stage 1: Build the Rust binary
|
||||
FROM rust:1.87-bookworm AS builder
|
||||
WORKDIR /build
|
||||
COPY . .
|
||||
RUN cargo build --release --bin blackwall \
|
||||
&& strip target/release/blackwall
|
||||
|
||||
# Stage 2: Minimal runtime image
|
||||
FROM debian:bookworm-slim
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
iproute2 \
|
||||
libelf1 \
|
||||
ca-certificates \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
COPY --from=builder /build/target/release/blackwall /usr/local/bin/blackwall
|
||||
RUN useradd -r -s /usr/sbin/nologin blackwall
|
||||
# eBPF requires root/CAP_BPF — runs as root in container, limited by securityContext
|
||||
ENTRYPOINT ["/usr/local/bin/blackwall"]
|
||||
CMD ["/etc/blackwall/config.toml"]
|
||||
16
deploy/docker/Dockerfile.ebpf
Executable file
16
deploy/docker/Dockerfile.ebpf
Executable file
|
|
@ -0,0 +1,16 @@
|
|||
# Blackwall eBPF programs — init container
|
||||
# Builds the BPF object file with nightly + bpfel target
|
||||
FROM rust:1.87-bookworm AS builder
|
||||
RUN rustup toolchain install nightly \
|
||||
&& rustup component add rust-src --toolchain nightly
|
||||
WORKDIR /build
|
||||
COPY . .
|
||||
RUN cd blackwall-ebpf && \
|
||||
cargo +nightly build \
|
||||
--target bpfel-unknown-none \
|
||||
-Z build-std=core \
|
||||
--release
|
||||
|
||||
# Stage 2: Tiny image with just the BPF binary
|
||||
FROM busybox:1.37
|
||||
COPY --from=builder /build/target/bpfel-unknown-none/release/blackwall-ebpf /opt/blackwall/blackwall-ebpf
|
||||
34
deploy/examples/blackwallpolicy-sample.yaml
Executable file
34
deploy/examples/blackwallpolicy-sample.yaml
Executable file
|
|
@ -0,0 +1,34 @@
|
|||
# Example BlackwallPolicy — drop known bad IPs, tarpit scanners
|
||||
apiVersion: security.blackwall.io/v1alpha1
|
||||
kind: BlackwallPolicy
|
||||
metadata:
|
||||
name: default-policy
|
||||
namespace: blackwall-system
|
||||
spec:
|
||||
rules:
|
||||
blocklist:
|
||||
- ip: "192.168.1.100"
|
||||
action: drop
|
||||
duration: "1h"
|
||||
- ip: "10.0.0.0/8"
|
||||
action: tarpit
|
||||
- ip: "203.0.113.0/24"
|
||||
action: drop
|
||||
allowlist:
|
||||
- ip: "192.168.0.0/16"
|
||||
reason: "internal network"
|
||||
thresholds:
|
||||
entropyAnomaly: 6500
|
||||
synFloodRate: 1000
|
||||
tarpit:
|
||||
enabled: true
|
||||
port: 2222
|
||||
baseDelayMs: 100
|
||||
maxDelayMs: 30000
|
||||
ai:
|
||||
enabled: true
|
||||
model: "qwen3:1.7b"
|
||||
fallbackModel: "qwen3:0.6b"
|
||||
network:
|
||||
interface: "auto"
|
||||
xdpMode: "native"
|
||||
61
deploy/healthcheck.sh
Executable file
61
deploy/healthcheck.sh
Executable file
|
|
@ -0,0 +1,61 @@
|
|||
#!/bin/bash
|
||||
# Blackwall Health Check — returns non-zero if any component is down
|
||||
|
||||
FAILED=0
|
||||
REPORT=""
|
||||
|
||||
# Check blackwall daemon
|
||||
if ! pidof blackwall > /dev/null 2>&1; then
|
||||
REPORT+="CRIT: blackwall not running\n"
|
||||
FAILED=1
|
||||
fi
|
||||
|
||||
# Check XDP attached
|
||||
if ! ip link show | grep -q xdp; then
|
||||
REPORT+="CRIT: XDP not attached to any interface\n"
|
||||
FAILED=1
|
||||
fi
|
||||
|
||||
# Check tarpit
|
||||
if ! pidof tarpit > /dev/null 2>&1; then
|
||||
REPORT+="WARN: tarpit not running\n"
|
||||
fi
|
||||
|
||||
# Check hivemind
|
||||
if ! pidof hivemind > /dev/null 2>&1; then
|
||||
REPORT+="WARN: hivemind not running\n"
|
||||
fi
|
||||
|
||||
# Check hivemind-api
|
||||
if ! ss -tlnp | grep -q 8090; then
|
||||
REPORT+="WARN: hivemind-api not listening on 8090\n"
|
||||
fi
|
||||
|
||||
# Check peer connectivity (if hivemind-api responds)
|
||||
STATS=$(curl -s --max-time 3 http://127.0.0.1:8090/stats 2>/dev/null)
|
||||
if [ -n "$STATS" ]; then
|
||||
PEERS=$(echo "$STATS" | grep -o '"peer_count":[0-9]*' | cut -d: -f2)
|
||||
if [ "${PEERS:-0}" -eq 0 ]; then
|
||||
REPORT+="WARN: hivemind has 0 peers\n"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Check Docker (if applicable)
|
||||
if command -v docker &> /dev/null; then
|
||||
DOCKER_COUNT=$(docker ps -q 2>/dev/null | wc -l)
|
||||
if [ "$DOCKER_COUNT" -eq 0 ]; then
|
||||
REPORT+="CRIT: No Docker containers running (expected >0)\n"
|
||||
FAILED=1
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ $FAILED -eq 1 ]; then
|
||||
echo -e "BLACKWALL HEALTH: CRITICAL\n$REPORT"
|
||||
exit 1
|
||||
elif [ -n "$REPORT" ]; then
|
||||
echo -e "BLACKWALL HEALTH: DEGRADED\n$REPORT"
|
||||
exit 0
|
||||
else
|
||||
echo "BLACKWALL HEALTH: OK"
|
||||
exit 0
|
||||
fi
|
||||
16
deploy/helm/blackwall/Chart.yaml
Executable file
16
deploy/helm/blackwall/Chart.yaml
Executable file
|
|
@ -0,0 +1,16 @@
|
|||
apiVersion: v2
|
||||
name: blackwall
|
||||
description: eBPF-powered AI firewall with XDP native DNAT, TCP tarpit deception, and P2P threat intelligence
|
||||
type: application
|
||||
version: 0.1.0
|
||||
appVersion: "2.0.0"
|
||||
keywords:
|
||||
- ebpf
|
||||
- xdp
|
||||
- firewall
|
||||
- honeypot
|
||||
- ai
|
||||
- security
|
||||
maintainers:
|
||||
- name: Blackwall Team
|
||||
home: https://github.com/blackwall-fw/blackwall
|
||||
36
deploy/helm/blackwall/templates/configmap.yaml
Executable file
36
deploy/helm/blackwall/templates/configmap.yaml
Executable file
|
|
@ -0,0 +1,36 @@
|
|||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-config
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
app.kubernetes.io/name: blackwall
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
data:
|
||||
config.toml: |
|
||||
[network]
|
||||
interface = {{ .Values.network.interface | quote }}
|
||||
xdp_mode = {{ .Values.network.xdpMode | quote }}
|
||||
|
||||
[tarpit]
|
||||
enabled = {{ .Values.tarpit.enabled }}
|
||||
port = {{ .Values.tarpit.port }}
|
||||
base_delay_ms = {{ .Values.tarpit.baseDelayMs }}
|
||||
max_delay_ms = {{ .Values.tarpit.maxDelayMs }}
|
||||
jitter_ms = {{ .Values.tarpit.jitterMs }}
|
||||
|
||||
[ai]
|
||||
enabled = {{ .Values.ai.enabled }}
|
||||
ollama_url = {{ .Values.ai.ollamaUrl | quote }}
|
||||
model = {{ .Values.ai.model | quote }}
|
||||
fallback_model = {{ .Values.ai.fallbackModel | quote }}
|
||||
max_tokens = {{ .Values.ai.maxTokens }}
|
||||
timeout_ms = {{ .Values.ai.timeoutMs }}
|
||||
|
||||
[feeds]
|
||||
enabled = {{ .Values.feeds.enabled }}
|
||||
refresh_interval_secs = {{ .Values.feeds.refreshIntervalSecs }}
|
||||
|
||||
[metrics]
|
||||
enabled = {{ .Values.metrics.enabled }}
|
||||
port = {{ .Values.metrics.port }}
|
||||
137
deploy/helm/blackwall/templates/crd-blackwallpolicy.yaml
Executable file
137
deploy/helm/blackwall/templates/crd-blackwallpolicy.yaml
Executable file
|
|
@ -0,0 +1,137 @@
|
|||
apiVersion: apiextensions.k8s.io/v1
|
||||
kind: CustomResourceDefinition
|
||||
metadata:
|
||||
name: blackwallpolicies.security.blackwall.io
|
||||
labels:
|
||||
app.kubernetes.io/name: blackwall
|
||||
spec:
|
||||
group: security.blackwall.io
|
||||
versions:
|
||||
- name: v1alpha1
|
||||
served: true
|
||||
storage: true
|
||||
schema:
|
||||
openAPIV3Schema:
|
||||
type: object
|
||||
properties:
|
||||
spec:
|
||||
type: object
|
||||
properties:
|
||||
# IP-based rules
|
||||
rules:
|
||||
type: object
|
||||
properties:
|
||||
blocklist:
|
||||
type: array
|
||||
items:
|
||||
type: object
|
||||
properties:
|
||||
ip:
|
||||
type: string
|
||||
pattern: '^(\d{1,3}\.){3}\d{1,3}(/\d{1,2})?$'
|
||||
action:
|
||||
type: string
|
||||
enum: ["drop", "tarpit", "pass"]
|
||||
duration:
|
||||
type: string
|
||||
pattern: '^\d+[smh]$'
|
||||
description: "Block duration (e.g., 10m, 1h, 30s)"
|
||||
required: ["ip", "action"]
|
||||
allowlist:
|
||||
type: array
|
||||
items:
|
||||
type: object
|
||||
properties:
|
||||
ip:
|
||||
type: string
|
||||
reason:
|
||||
type: string
|
||||
required: ["ip"]
|
||||
# Anomaly detection thresholds
|
||||
thresholds:
|
||||
type: object
|
||||
properties:
|
||||
entropyAnomaly:
|
||||
type: integer
|
||||
minimum: 0
|
||||
maximum: 8000
|
||||
description: "Byte diversity score above which packets are flagged"
|
||||
synFloodRate:
|
||||
type: integer
|
||||
minimum: 0
|
||||
description: "SYN packets per second before triggering protection"
|
||||
# Tarpit configuration
|
||||
tarpit:
|
||||
type: object
|
||||
properties:
|
||||
enabled:
|
||||
type: boolean
|
||||
port:
|
||||
type: integer
|
||||
minimum: 1
|
||||
maximum: 65535
|
||||
baseDelayMs:
|
||||
type: integer
|
||||
maxDelayMs:
|
||||
type: integer
|
||||
# AI classification
|
||||
ai:
|
||||
type: object
|
||||
properties:
|
||||
enabled:
|
||||
type: boolean
|
||||
model:
|
||||
type: string
|
||||
fallbackModel:
|
||||
type: string
|
||||
# Interface selection (per-node override via nodeSelector)
|
||||
network:
|
||||
type: object
|
||||
properties:
|
||||
interface:
|
||||
type: string
|
||||
xdpMode:
|
||||
type: string
|
||||
enum: ["generic", "native", "offload"]
|
||||
status:
|
||||
type: object
|
||||
properties:
|
||||
conditions:
|
||||
type: array
|
||||
items:
|
||||
type: object
|
||||
properties:
|
||||
type:
|
||||
type: string
|
||||
status:
|
||||
type: string
|
||||
lastTransitionTime:
|
||||
type: string
|
||||
format: date-time
|
||||
message:
|
||||
type: string
|
||||
appliedNodes:
|
||||
type: integer
|
||||
description: "Number of nodes where policy is active"
|
||||
blockedIPs:
|
||||
type: integer
|
||||
description: "Total IPs currently in blocklist across cluster"
|
||||
subresources:
|
||||
status: {}
|
||||
additionalPrinterColumns:
|
||||
- name: Rules
|
||||
type: integer
|
||||
jsonPath: .status.blockedIPs
|
||||
- name: Nodes
|
||||
type: integer
|
||||
jsonPath: .status.appliedNodes
|
||||
- name: Age
|
||||
type: date
|
||||
jsonPath: .metadata.creationTimestamp
|
||||
scope: Namespaced
|
||||
names:
|
||||
plural: blackwallpolicies
|
||||
singular: blackwallpolicy
|
||||
kind: BlackwallPolicy
|
||||
shortNames:
|
||||
- bwp
|
||||
119
deploy/helm/blackwall/templates/daemonset.yaml
Executable file
119
deploy/helm/blackwall/templates/daemonset.yaml
Executable file
|
|
@ -0,0 +1,119 @@
|
|||
apiVersion: apps/v1
|
||||
kind: DaemonSet
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-blackwall
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
app.kubernetes.io/name: blackwall
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
|
||||
app.kubernetes.io/component: dataplane
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: blackwall
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
updateStrategy:
|
||||
type: RollingUpdate
|
||||
rollingUpdate:
|
||||
maxUnavailable: 1
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: blackwall
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
annotations:
|
||||
prometheus.io/scrape: "true"
|
||||
prometheus.io/port: {{ .Values.metrics.port | quote }}
|
||||
spec:
|
||||
{{- if .Values.serviceAccount.create }}
|
||||
serviceAccountName: {{ .Values.serviceAccount.name }}
|
||||
{{- end }}
|
||||
hostNetwork: true
|
||||
dnsPolicy: ClusterFirstWithHostNet
|
||||
{{- with .Values.nodeSelector }}
|
||||
nodeSelector:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- with .Values.tolerations }}
|
||||
tolerations:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
initContainers:
|
||||
# Copy pre-built eBPF object into shared volume
|
||||
- name: ebpf-loader
|
||||
image: "{{ .Values.ebpfImage.repository }}:{{ .Values.ebpfImage.tag }}"
|
||||
imagePullPolicy: {{ .Values.ebpfImage.pullPolicy }}
|
||||
command: ["cp", "/opt/blackwall/blackwall-ebpf", "/bpf/blackwall-ebpf"]
|
||||
volumeMounts:
|
||||
- name: bpf-objects
|
||||
mountPath: /bpf
|
||||
containers:
|
||||
- name: blackwall
|
||||
image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
|
||||
imagePullPolicy: {{ .Values.image.pullPolicy }}
|
||||
args:
|
||||
- "/etc/blackwall/config.toml"
|
||||
env:
|
||||
- name: BLACKWALL_EBPF_PATH
|
||||
value: "/bpf/blackwall-ebpf"
|
||||
- name: RUST_LOG
|
||||
value: "blackwall=info"
|
||||
- name: NODE_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: spec.nodeName
|
||||
securityContext:
|
||||
{{- toYaml .Values.securityContext | nindent 12 }}
|
||||
resources:
|
||||
{{- toYaml .Values.resources | nindent 12 }}
|
||||
volumeMounts:
|
||||
- name: bpf-objects
|
||||
mountPath: /bpf
|
||||
readOnly: true
|
||||
- name: config
|
||||
mountPath: /etc/blackwall
|
||||
readOnly: true
|
||||
- name: bpf-fs
|
||||
mountPath: /sys/fs/bpf
|
||||
{{- if .Values.pcap.enabled }}
|
||||
- name: pcap-storage
|
||||
mountPath: /var/lib/blackwall/pcap
|
||||
{{- end }}
|
||||
ports:
|
||||
{{- if .Values.tarpit.enabled }}
|
||||
- name: tarpit
|
||||
containerPort: {{ .Values.tarpit.port }}
|
||||
protocol: TCP
|
||||
hostPort: {{ .Values.tarpit.port }}
|
||||
{{- end }}
|
||||
{{- if .Values.metrics.enabled }}
|
||||
- name: metrics
|
||||
containerPort: {{ .Values.metrics.port }}
|
||||
protocol: TCP
|
||||
{{- end }}
|
||||
livenessProbe:
|
||||
exec:
|
||||
command: ["cat", "/proc/1/status"]
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 30
|
||||
readinessProbe:
|
||||
exec:
|
||||
command: ["test", "-f", "/sys/fs/bpf/blackwall_xdp"]
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 10
|
||||
volumes:
|
||||
- name: bpf-objects
|
||||
emptyDir: {}
|
||||
- name: config
|
||||
configMap:
|
||||
name: {{ .Release.Name }}-config
|
||||
- name: bpf-fs
|
||||
hostPath:
|
||||
path: /sys/fs/bpf
|
||||
type: DirectoryOrCreate
|
||||
{{- if .Values.pcap.enabled }}
|
||||
- name: pcap-storage
|
||||
persistentVolumeClaim:
|
||||
claimName: {{ .Release.Name }}-pcap
|
||||
{{- end }}
|
||||
39
deploy/helm/blackwall/templates/rbac.yaml
Executable file
39
deploy/helm/blackwall/templates/rbac.yaml
Executable file
|
|
@ -0,0 +1,39 @@
|
|||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-blackwall
|
||||
labels:
|
||||
app.kubernetes.io/name: blackwall
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
rules:
|
||||
# Watch BlackwallPolicy CRDs
|
||||
- apiGroups: ["security.blackwall.io"]
|
||||
resources: ["blackwallpolicies"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: ["security.blackwall.io"]
|
||||
resources: ["blackwallpolicies/status"]
|
||||
verbs: ["patch", "update"]
|
||||
# Read node info for interface auto-detection
|
||||
- apiGroups: [""]
|
||||
resources: ["nodes"]
|
||||
verbs: ["get", "list"]
|
||||
# ConfigMaps for config
|
||||
- apiGroups: [""]
|
||||
resources: ["configmaps"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-blackwall
|
||||
labels:
|
||||
app.kubernetes.io/name: blackwall
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: {{ .Release.Name }}-blackwall
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: {{ .Values.serviceAccount.name }}
|
||||
namespace: {{ .Release.Namespace }}
|
||||
14
deploy/helm/blackwall/templates/serviceaccount.yaml
Executable file
14
deploy/helm/blackwall/templates/serviceaccount.yaml
Executable file
|
|
@ -0,0 +1,14 @@
|
|||
{{- if .Values.serviceAccount.create }}
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: {{ .Values.serviceAccount.name }}
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
app.kubernetes.io/name: blackwall
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
{{- with .Values.serviceAccount.annotations }}
|
||||
annotations:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
92
deploy/helm/blackwall/values.yaml
Executable file
92
deploy/helm/blackwall/values.yaml
Executable file
|
|
@ -0,0 +1,92 @@
|
|||
# Blackwall Helm Chart — Default Values
|
||||
# Override per environment via -f values-production.yaml
|
||||
|
||||
# Container image
|
||||
image:
|
||||
repository: ghcr.io/blackwall-fw/blackwall
|
||||
tag: "2.0.0"
|
||||
pullPolicy: IfNotPresent
|
||||
|
||||
# eBPF image (init container builds/copies the BPF object)
|
||||
ebpfImage:
|
||||
repository: ghcr.io/blackwall-fw/blackwall-ebpf
|
||||
tag: "2.0.0"
|
||||
pullPolicy: IfNotPresent
|
||||
|
||||
# DaemonSet scheduling
|
||||
nodeSelector: {}
|
||||
tolerations:
|
||||
- key: node-role.kubernetes.io/master
|
||||
effect: NoSchedule
|
||||
affinity: {}
|
||||
|
||||
# Resource limits
|
||||
resources:
|
||||
limits:
|
||||
cpu: "500m"
|
||||
memory: "256Mi"
|
||||
requests:
|
||||
cpu: "100m"
|
||||
memory: "128Mi"
|
||||
|
||||
# Network configuration
|
||||
network:
|
||||
# Interface to attach XDP program to.
|
||||
# "auto" = detect primary interface via default route.
|
||||
interface: "auto"
|
||||
# XDP attach mode: generic, native, offload
|
||||
xdpMode: "generic"
|
||||
|
||||
# Tarpit honeypot
|
||||
tarpit:
|
||||
enabled: true
|
||||
port: 2222
|
||||
baseDelayMs: 100
|
||||
maxDelayMs: 30000
|
||||
jitterMs: 500
|
||||
|
||||
# AI/LLM classification
|
||||
ai:
|
||||
enabled: true
|
||||
ollamaUrl: "http://ollama.default.svc.cluster.local:11434"
|
||||
model: "qwen3:1.7b"
|
||||
fallbackModel: "qwen3:0.6b"
|
||||
maxTokens: 512
|
||||
timeoutMs: 30000
|
||||
|
||||
# Threat feeds
|
||||
feeds:
|
||||
enabled: true
|
||||
refreshIntervalSecs: 3600
|
||||
|
||||
# HiveMind P2P mesh
|
||||
hivemind:
|
||||
enabled: false
|
||||
bootstrapPeers: []
|
||||
|
||||
# PCAP forensic capture
|
||||
pcap:
|
||||
enabled: false
|
||||
storageClass: ""
|
||||
storageSize: "10Gi"
|
||||
|
||||
# Metrics (Prometheus)
|
||||
metrics:
|
||||
enabled: true
|
||||
port: 9090
|
||||
|
||||
# ServiceAccount
|
||||
serviceAccount:
|
||||
create: true
|
||||
name: "blackwall"
|
||||
annotations: {}
|
||||
|
||||
# Security context (eBPF requires CAP_BPF + CAP_NET_ADMIN)
|
||||
securityContext:
|
||||
privileged: false
|
||||
capabilities:
|
||||
add:
|
||||
- BPF
|
||||
- NET_ADMIN
|
||||
- SYS_ADMIN
|
||||
- PERFMON
|
||||
14
deploy/systemd/laptop/blackwall-api.service
Executable file
14
deploy/systemd/laptop/blackwall-api.service
Executable file
|
|
@ -0,0 +1,14 @@
|
|||
[Unit]
|
||||
Description=Blackwall HiveMind API
|
||||
After=blackwall-hivemind.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
WorkingDirectory=/opt/blackwall
|
||||
ExecStart=/opt/blackwall/bin/hivemind-api
|
||||
Environment=RUST_LOG=info
|
||||
Restart=on-failure
|
||||
RestartSec=3
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
8
deploy/systemd/laptop/blackwall-healthcheck.service
Executable file
8
deploy/systemd/laptop/blackwall-healthcheck.service
Executable file
|
|
@ -0,0 +1,8 @@
|
|||
[Unit]
|
||||
Description=Blackwall health check
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/opt/blackwall/healthcheck.sh
|
||||
StandardOutput=append:/var/log/blackwall-health.log
|
||||
StandardError=append:/var/log/blackwall-health.log
|
||||
10
deploy/systemd/laptop/blackwall-healthcheck.timer
Executable file
10
deploy/systemd/laptop/blackwall-healthcheck.timer
Executable file
|
|
@ -0,0 +1,10 @@
|
|||
[Unit]
|
||||
Description=Blackwall health check timer (every 5 min)
|
||||
|
||||
[Timer]
|
||||
OnBootSec=2min
|
||||
OnUnitActiveSec=5min
|
||||
Persistent=true
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
14
deploy/systemd/laptop/blackwall-hivemind.service
Executable file
14
deploy/systemd/laptop/blackwall-hivemind.service
Executable file
|
|
@ -0,0 +1,14 @@
|
|||
[Unit]
|
||||
Description=Blackwall HiveMind P2P Mesh
|
||||
After=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
WorkingDirectory=/opt/blackwall
|
||||
ExecStart=/opt/blackwall/bin/hivemind
|
||||
Environment=RUST_LOG=info
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
14
deploy/systemd/laptop/blackwall-tarpit.service
Executable file
14
deploy/systemd/laptop/blackwall-tarpit.service
Executable file
|
|
@ -0,0 +1,14 @@
|
|||
[Unit]
|
||||
Description=Blackwall Tarpit Honeypot
|
||||
After=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
WorkingDirectory=/opt/blackwall
|
||||
ExecStart=/opt/blackwall/bin/tarpit
|
||||
Environment=RUST_LOG=info
|
||||
Restart=on-failure
|
||||
RestartSec=3
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
18
deploy/systemd/laptop/blackwall.service
Executable file
18
deploy/systemd/laptop/blackwall.service
Executable file
|
|
@ -0,0 +1,18 @@
|
|||
[Unit]
|
||||
Description=Blackwall Adaptive eBPF Firewall
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
WorkingDirectory=/opt/blackwall
|
||||
ExecStart=/opt/blackwall/bin/blackwall /opt/blackwall/config.toml
|
||||
Environment=BLACKWALL_EBPF_PATH=/opt/blackwall/bin/blackwall-ebpf
|
||||
Environment=RUST_LOG=info
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
LimitMEMLOCK=infinity
|
||||
AmbientCapabilities=CAP_BPF CAP_NET_ADMIN CAP_PERFMON CAP_SYS_PTRACE
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
14
deploy/systemd/server/blackwall-api.service
Executable file
14
deploy/systemd/server/blackwall-api.service
Executable file
|
|
@ -0,0 +1,14 @@
|
|||
[Unit]
|
||||
Description=Blackwall HiveMind API
|
||||
After=blackwall-hivemind.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
WorkingDirectory=/opt/blackwall
|
||||
ExecStart=/opt/blackwall/hivemind-api
|
||||
Environment=RUST_LOG=info
|
||||
Restart=on-failure
|
||||
RestartSec=3
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
14
deploy/systemd/server/blackwall-hivemind.service
Executable file
14
deploy/systemd/server/blackwall-hivemind.service
Executable file
|
|
@ -0,0 +1,14 @@
|
|||
[Unit]
|
||||
Description=Blackwall HiveMind P2P Mesh
|
||||
After=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
WorkingDirectory=/opt/blackwall
|
||||
ExecStart=/usr/local/bin/hivemind
|
||||
Environment=RUST_LOG=info
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
14
deploy/systemd/server/blackwall-tarpit.service
Executable file
14
deploy/systemd/server/blackwall-tarpit.service
Executable file
|
|
@ -0,0 +1,14 @@
|
|||
[Unit]
|
||||
Description=Blackwall Tarpit Honeypot
|
||||
After=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
WorkingDirectory=/opt/blackwall
|
||||
ExecStart=/opt/blackwall/tarpit
|
||||
Environment=RUST_LOG=info
|
||||
Restart=on-failure
|
||||
RestartSec=3
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
18
deploy/systemd/server/blackwall.service
Executable file
18
deploy/systemd/server/blackwall.service
Executable file
|
|
@ -0,0 +1,18 @@
|
|||
[Unit]
|
||||
Description=Blackwall Adaptive eBPF Firewall
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
WorkingDirectory=/opt/blackwall
|
||||
ExecStart=/opt/blackwall/blackwall /opt/blackwall/config.toml
|
||||
Environment=BLACKWALL_EBPF_PATH=/opt/blackwall/blackwall-ebpf
|
||||
Environment=RUST_LOG=info
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
LimitMEMLOCK=infinity
|
||||
AmbientCapabilities=CAP_BPF CAP_NET_ADMIN CAP_PERFMON CAP_SYS_PTRACE
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
22
hivemind-api/Cargo.toml
Executable file
22
hivemind-api/Cargo.toml
Executable file
|
|
@ -0,0 +1,22 @@
|
|||
[package]
|
||||
name = "hivemind-api"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
description = "Enterprise Threat Feed API — REST/STIX/TAXII endpoint for HiveMind verified IoCs"
|
||||
|
||||
[dependencies]
|
||||
common = { path = "../common", default-features = false, features = ["user"] }
|
||||
tokio = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
tracing-subscriber = { workspace = true }
|
||||
anyhow = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
hyper = { workspace = true, features = ["server"] }
|
||||
hyper-util = { workspace = true }
|
||||
http-body-util = { workspace = true }
|
||||
ring = { workspace = true }
|
||||
|
||||
[[bin]]
|
||||
name = "hivemind-api"
|
||||
path = "src/main.rs"
|
||||
305
hivemind-api/src/feed.rs
Executable file
305
hivemind-api/src/feed.rs
Executable file
|
|
@ -0,0 +1,305 @@
|
|||
/// Feed query parameter parsing and response formatting.
|
||||
///
|
||||
/// Bridges the HTTP layer (server.rs) to the storage layer (store.rs)
|
||||
/// by parsing URL query parameters into `QueryParams` and formatting
|
||||
/// paginated feed results as JSON responses.
|
||||
use common::hivemind;
|
||||
|
||||
use crate::store::QueryParams;
|
||||
|
||||
/// Parse query parameters from a URI query string.
|
||||
///
|
||||
/// Supported parameters:
|
||||
/// - `since` — Unix timestamp filter (only IoCs verified after this time)
|
||||
/// - `severity` — Minimum severity level (0-4)
|
||||
/// - `type` — IoC type filter (0-4)
|
||||
/// - `limit` — Page size (capped by tier max)
|
||||
/// - `offset` — Pagination offset
|
||||
///
|
||||
/// Invalid parameter values are silently ignored (defaults used).
|
||||
pub fn parse_query_params(query: Option<&str>, max_page_size: usize) -> QueryParams {
|
||||
let mut params = QueryParams::new();
|
||||
params.limit = params.limit.min(max_page_size);
|
||||
|
||||
let query = match query {
|
||||
Some(q) => q,
|
||||
None => return params,
|
||||
};
|
||||
|
||||
for pair in query.split('&') {
|
||||
let mut parts = pair.splitn(2, '=');
|
||||
let key = match parts.next() {
|
||||
Some(k) => k,
|
||||
None => continue,
|
||||
};
|
||||
let value = match parts.next() {
|
||||
Some(v) => v,
|
||||
None => continue,
|
||||
};
|
||||
|
||||
match key {
|
||||
"since" => {
|
||||
if let Ok(ts) = value.parse::<u64>() {
|
||||
params.since = Some(ts);
|
||||
}
|
||||
}
|
||||
"severity" => {
|
||||
if let Ok(sev) = value.parse::<u8>() {
|
||||
if sev <= 4 {
|
||||
params.min_severity = Some(sev);
|
||||
}
|
||||
}
|
||||
}
|
||||
"type" => {
|
||||
if let Ok(t) = value.parse::<u8>() {
|
||||
if t <= 4 {
|
||||
params.ioc_type = Some(t);
|
||||
}
|
||||
}
|
||||
}
|
||||
"limit" => {
|
||||
if let Ok(l) = value.parse::<usize>() {
|
||||
params.limit = l.min(max_page_size).max(1);
|
||||
}
|
||||
}
|
||||
"offset" => {
|
||||
if let Ok(o) = value.parse::<usize>() {
|
||||
params.offset = o;
|
||||
}
|
||||
}
|
||||
_ => {} // Unknown params silently ignored
|
||||
}
|
||||
}
|
||||
|
||||
params
|
||||
}
|
||||
|
||||
/// Feed statistics for the /api/v1/stats endpoint.
|
||||
#[derive(Clone, Debug, serde::Serialize)]
|
||||
pub struct FeedStats {
|
||||
/// Total verified IoCs in the feed.
|
||||
pub total_iocs: usize,
|
||||
/// Breakdown by severity level.
|
||||
pub by_severity: SeverityBreakdown,
|
||||
/// Breakdown by IoC type.
|
||||
pub by_type: TypeBreakdown,
|
||||
}
|
||||
|
||||
/// Count of IoCs per severity level.
|
||||
#[derive(Clone, Debug, Default, serde::Serialize)]
|
||||
pub struct SeverityBreakdown {
|
||||
pub info: usize,
|
||||
pub low: usize,
|
||||
pub medium: usize,
|
||||
pub high: usize,
|
||||
pub critical: usize,
|
||||
}
|
||||
|
||||
/// Count of IoCs per type.
|
||||
#[derive(Clone, Debug, Default, serde::Serialize)]
|
||||
pub struct TypeBreakdown {
|
||||
pub malicious_ip: usize,
|
||||
pub ja4_fingerprint: usize,
|
||||
pub entropy_anomaly: usize,
|
||||
pub dns_tunnel: usize,
|
||||
pub behavioral_pattern: usize,
|
||||
}
|
||||
|
||||
/// Compute feed statistics from the store.
|
||||
pub fn compute_stats(store: &crate::store::ThreatFeedStore) -> FeedStats {
|
||||
let all = store.all();
|
||||
let mut by_severity = SeverityBreakdown::default();
|
||||
let mut by_type = TypeBreakdown::default();
|
||||
|
||||
for vioc in all {
|
||||
match hivemind::ThreatSeverity::from_u8(vioc.ioc.severity) {
|
||||
hivemind::ThreatSeverity::Info => by_severity.info += 1,
|
||||
hivemind::ThreatSeverity::Low => by_severity.low += 1,
|
||||
hivemind::ThreatSeverity::Medium => by_severity.medium += 1,
|
||||
hivemind::ThreatSeverity::High => by_severity.high += 1,
|
||||
hivemind::ThreatSeverity::Critical => by_severity.critical += 1,
|
||||
}
|
||||
|
||||
match hivemind::IoCType::from_u8(vioc.ioc.ioc_type) {
|
||||
hivemind::IoCType::MaliciousIp => by_type.malicious_ip += 1,
|
||||
hivemind::IoCType::Ja4Fingerprint => by_type.ja4_fingerprint += 1,
|
||||
hivemind::IoCType::EntropyAnomaly => by_type.entropy_anomaly += 1,
|
||||
hivemind::IoCType::DnsTunnel => by_type.dns_tunnel += 1,
|
||||
hivemind::IoCType::BehavioralPattern => by_type.behavioral_pattern += 1,
|
||||
}
|
||||
}
|
||||
|
||||
FeedStats {
|
||||
total_iocs: all.len(),
|
||||
by_severity,
|
||||
by_type,
|
||||
}
|
||||
}
|
||||
|
||||
/// Mesh stats compatible with the TUI dashboard's MeshStats struct.
|
||||
///
|
||||
/// Returns IoC counts mapped to the dashboard's expected fields.
|
||||
#[derive(Clone, Debug, serde::Serialize)]
|
||||
pub struct DashboardMeshStats {
|
||||
pub connected: bool,
|
||||
|
||||
// P2P Mesh
|
||||
pub peer_count: u64,
|
||||
pub dht_records: u64,
|
||||
pub gossip_topics: u64,
|
||||
pub messages_per_sec: f64,
|
||||
|
||||
// Threat Intel
|
||||
pub iocs_shared: u64,
|
||||
pub iocs_received: u64,
|
||||
pub avg_reputation: f64,
|
||||
|
||||
// Network Firewall (XDP/eBPF)
|
||||
pub packets_total: u64,
|
||||
pub packets_passed: u64,
|
||||
pub packets_dropped: u64,
|
||||
pub anomalies_sent: u64,
|
||||
|
||||
// A2A Firewall (separate from XDP)
|
||||
pub a2a_jwts_verified: u64,
|
||||
pub a2a_violations: u64,
|
||||
pub a2a_injections: u64,
|
||||
|
||||
// Cryptography
|
||||
pub zkp_proofs_generated: u64,
|
||||
pub zkp_proofs_verified: u64,
|
||||
pub fhe_encrypted: bool,
|
||||
}
|
||||
|
||||
/// Compute dashboard-compatible mesh stats from the store.
|
||||
pub fn compute_mesh_stats(store: &crate::store::ThreatFeedStore, counters: &crate::server::HivemindCounters) -> DashboardMeshStats {
|
||||
use std::sync::atomic::Ordering;
|
||||
let all = store.all();
|
||||
let total = all.len() as u64;
|
||||
|
||||
// eBPF/XDP counters
|
||||
let pkt_total = counters.packets_total.load(Ordering::Relaxed);
|
||||
let pkt_passed = counters.packets_passed.load(Ordering::Relaxed);
|
||||
let pkt_dropped = counters.packets_dropped.load(Ordering::Relaxed);
|
||||
let anomalies = counters.anomalies_sent.load(Ordering::Relaxed);
|
||||
|
||||
// P2P counters
|
||||
let peers = counters.peer_count.load(Ordering::Relaxed);
|
||||
let iocs_p2p = counters.iocs_shared_p2p.load(Ordering::Relaxed);
|
||||
let rep_x100 = counters.avg_reputation_x100.load(Ordering::Relaxed);
|
||||
let msgs_total = counters.messages_total.load(Ordering::Relaxed);
|
||||
|
||||
// A2A counters
|
||||
let a2a_jwts = counters.a2a_jwts_verified.load(Ordering::Relaxed);
|
||||
let a2a_viol = counters.a2a_violations.load(Ordering::Relaxed);
|
||||
let a2a_inj = counters.a2a_injections.load(Ordering::Relaxed);
|
||||
|
||||
DashboardMeshStats {
|
||||
connected: true,
|
||||
peer_count: peers,
|
||||
dht_records: total,
|
||||
gossip_topics: if total > 0 || peers > 0 { 1 } else { 0 },
|
||||
messages_per_sec: msgs_total as f64 / 60.0,
|
||||
iocs_shared: iocs_p2p,
|
||||
iocs_received: pkt_total,
|
||||
avg_reputation: rep_x100 as f64 / 100.0,
|
||||
packets_total: pkt_total,
|
||||
packets_passed: pkt_passed,
|
||||
packets_dropped: pkt_dropped,
|
||||
anomalies_sent: anomalies,
|
||||
a2a_jwts_verified: a2a_jwts,
|
||||
a2a_violations: a2a_viol,
|
||||
a2a_injections: a2a_inj,
|
||||
zkp_proofs_generated: 0,
|
||||
zkp_proofs_verified: 0,
|
||||
fhe_encrypted: false,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn parse_empty_query() {
|
||||
let params = parse_query_params(None, 1000);
|
||||
assert_eq!(params.limit, hivemind::API_DEFAULT_PAGE_SIZE);
|
||||
assert_eq!(params.offset, 0);
|
||||
assert!(params.since.is_none());
|
||||
assert!(params.min_severity.is_none());
|
||||
assert!(params.ioc_type.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_all_params() {
|
||||
let params = parse_query_params(
|
||||
Some("since=1700000000&severity=3&type=1&limit=50&offset=10"),
|
||||
1000,
|
||||
);
|
||||
assert_eq!(params.since, Some(1700000000));
|
||||
assert_eq!(params.min_severity, Some(3));
|
||||
assert_eq!(params.ioc_type, Some(1));
|
||||
assert_eq!(params.limit, 50);
|
||||
assert_eq!(params.offset, 10);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn limit_capped_by_tier() {
|
||||
let params = parse_query_params(Some("limit=5000"), 100);
|
||||
assert_eq!(params.limit, 100);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_params_ignored() {
|
||||
let params = parse_query_params(
|
||||
Some("since=notanumber&severity=99&limit=abc&unknown=foo"),
|
||||
1000,
|
||||
);
|
||||
assert!(params.since.is_none());
|
||||
assert!(params.min_severity.is_none()); // 99 > 4, ignored
|
||||
assert_eq!(params.limit, hivemind::API_DEFAULT_PAGE_SIZE);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compute_stats_populated() {
|
||||
use crate::store::ThreatFeedStore;
|
||||
use common::hivemind::IoC;
|
||||
|
||||
let mut store = ThreatFeedStore::new();
|
||||
store.insert(
|
||||
IoC {
|
||||
ioc_type: 0,
|
||||
severity: 3,
|
||||
ip: 1,
|
||||
ja4: None,
|
||||
entropy_score: None,
|
||||
description: "test".to_string(),
|
||||
first_seen: 1000,
|
||||
confirmations: 3,
|
||||
zkp_proof: Vec::new(),
|
||||
},
|
||||
2000,
|
||||
);
|
||||
store.insert(
|
||||
IoC {
|
||||
ioc_type: 1,
|
||||
severity: 4,
|
||||
ip: 2,
|
||||
ja4: None,
|
||||
entropy_score: None,
|
||||
description: "test2".to_string(),
|
||||
first_seen: 1000,
|
||||
confirmations: 3,
|
||||
zkp_proof: Vec::new(),
|
||||
},
|
||||
3000,
|
||||
);
|
||||
|
||||
let stats = compute_stats(&store);
|
||||
assert_eq!(stats.total_iocs, 2);
|
||||
assert_eq!(stats.by_severity.high, 1);
|
||||
assert_eq!(stats.by_severity.critical, 1);
|
||||
assert_eq!(stats.by_type.malicious_ip, 1);
|
||||
assert_eq!(stats.by_type.ja4_fingerprint, 1);
|
||||
}
|
||||
}
|
||||
187
hivemind-api/src/integrations/cef.rs
Executable file
187
hivemind-api/src/integrations/cef.rs
Executable file
|
|
@ -0,0 +1,187 @@
|
|||
/// ArcSight Common Event Format (CEF) exporter.
|
||||
///
|
||||
/// Converts verified IoCs to CEF format for ingestion by ArcSight,
|
||||
/// Sentinel, and other SIEM platforms that support CEF.
|
||||
///
|
||||
/// Format: `CEF:0|Vendor|Product|Version|SignatureID|Name|Severity|Extensions`
|
||||
///
|
||||
/// Reference: <https://www.microfocus.com/documentation/arcsight/arcsight-smartconnectors/pdfdoc/cef-implementation-standard/cef-implementation-standard.pdf>
|
||||
use common::hivemind::{self, IoCType, ThreatSeverity};
|
||||
|
||||
use crate::store::{ip_to_string, unix_to_iso8601, VerifiedIoC};
|
||||
|
||||
/// Convert a verified IoC to a CEF format string.
|
||||
///
|
||||
/// The returned string is a single CEF event line.
|
||||
pub fn ioc_to_cef(vioc: &VerifiedIoC) -> String {
|
||||
let ioc = &vioc.ioc;
|
||||
let ioc_type = IoCType::from_u8(ioc.ioc_type);
|
||||
let severity = ThreatSeverity::from_u8(ioc.severity);
|
||||
|
||||
let sig_id = cef_signature_id(ioc_type);
|
||||
let name = escape_cef_header(&cef_event_name(ioc_type, ioc));
|
||||
let sev = cef_severity(severity);
|
||||
|
||||
// CEF header (pipe-delimited, 7 fields after CEF:0)
|
||||
let header = format!(
|
||||
"CEF:0|{}|{}|{}|{sig_id}|{name}|{sev}",
|
||||
escape_cef_header(hivemind::SIEM_VENDOR),
|
||||
escape_cef_header(hivemind::SIEM_PRODUCT),
|
||||
escape_cef_header(hivemind::SIEM_VERSION),
|
||||
);
|
||||
|
||||
// CEF extensions (key=value space-delimited)
|
||||
let src_ip = ip_to_string(ioc.ip);
|
||||
let timestamp = unix_to_iso8601(vioc.verified_at);
|
||||
|
||||
let mut ext = format!(
|
||||
"src={src_ip} rt={timestamp} cat={} msg={} cs1Label=stix_id cs1={} \
|
||||
cn1Label=confirmations cn1={}",
|
||||
escape_cef_value(cef_category(ioc_type)),
|
||||
escape_cef_value(&ioc.description),
|
||||
escape_cef_value(&vioc.stix_id),
|
||||
ioc.confirmations,
|
||||
);
|
||||
|
||||
if let Some(ref ja4) = ioc.ja4 {
|
||||
ext.push_str(&format!(
|
||||
" cs2Label=ja4 cs2={}",
|
||||
escape_cef_value(ja4)
|
||||
));
|
||||
}
|
||||
|
||||
if let Some(entropy) = ioc.entropy_score {
|
||||
ext.push_str(&format!(" cn2Label=entropy_score cn2={entropy}"));
|
||||
}
|
||||
|
||||
format!("{header}|{ext}")
|
||||
}
|
||||
|
||||
/// Convert a batch of verified IoCs to newline-delimited CEF.
|
||||
pub fn batch_to_cef(iocs: &[VerifiedIoC]) -> String {
|
||||
iocs.iter()
|
||||
.map(ioc_to_cef)
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n")
|
||||
}
|
||||
|
||||
/// Map IoC type to CEF signature ID.
|
||||
fn cef_signature_id(t: IoCType) -> u16 {
|
||||
match t {
|
||||
IoCType::MaliciousIp => 1001,
|
||||
IoCType::Ja4Fingerprint => 1002,
|
||||
IoCType::EntropyAnomaly => 1003,
|
||||
IoCType::DnsTunnel => 1004,
|
||||
IoCType::BehavioralPattern => 1005,
|
||||
}
|
||||
}
|
||||
|
||||
/// Build human-readable CEF event name.
|
||||
fn cef_event_name(t: IoCType, ioc: &common::hivemind::IoC) -> String {
|
||||
match t {
|
||||
IoCType::MaliciousIp => format!("Malicious IP {}", ip_to_string(ioc.ip)),
|
||||
IoCType::Ja4Fingerprint => "Malicious TLS Fingerprint".to_string(),
|
||||
IoCType::EntropyAnomaly => "High Entropy Anomaly".to_string(),
|
||||
IoCType::DnsTunnel => "DNS Tunneling Detected".to_string(),
|
||||
IoCType::BehavioralPattern => "Behavioral Anomaly".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Map threat severity to CEF severity (0-10).
|
||||
fn cef_severity(s: ThreatSeverity) -> u8 {
|
||||
match s {
|
||||
ThreatSeverity::Info => 1,
|
||||
ThreatSeverity::Low => 3,
|
||||
ThreatSeverity::Medium => 5,
|
||||
ThreatSeverity::High => 8,
|
||||
ThreatSeverity::Critical => 10,
|
||||
}
|
||||
}
|
||||
|
||||
/// Map IoC type to CEF category string.
|
||||
fn cef_category(t: IoCType) -> &'static str {
|
||||
match t {
|
||||
IoCType::MaliciousIp => "Threat/MaliciousIP",
|
||||
IoCType::Ja4Fingerprint => "Threat/TLSFingerprint",
|
||||
IoCType::EntropyAnomaly => "Anomaly/Entropy",
|
||||
IoCType::DnsTunnel => "Threat/DNSTunnel",
|
||||
IoCType::BehavioralPattern => "Anomaly/Behavioral",
|
||||
}
|
||||
}
|
||||
|
||||
/// Escape pipe characters in CEF header fields.
|
||||
///
|
||||
/// CEF uses `|` as the header delimiter — pipes must be escaped as `\|`.
|
||||
fn escape_cef_header(s: &str) -> String {
|
||||
s.replace('\\', "\\\\").replace('|', "\\|")
|
||||
}
|
||||
|
||||
/// Escape special characters in CEF extension values.
|
||||
///
|
||||
/// Backslash, equals, and newlines must be escaped in extension values.
|
||||
fn escape_cef_value(s: &str) -> String {
|
||||
s.replace('\\', "\\\\")
|
||||
.replace('=', "\\=")
|
||||
.replace('\n', "\\n")
|
||||
.replace('\r', "\\r")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use common::hivemind::IoC;
|
||||
|
||||
fn sample_vioc() -> VerifiedIoC {
|
||||
VerifiedIoC {
|
||||
ioc: IoC {
|
||||
ioc_type: 0,
|
||||
severity: 4,
|
||||
ip: 0x0A000001,
|
||||
ja4: Some("t13d".to_string()),
|
||||
entropy_score: Some(8000),
|
||||
description: "Critical threat".to_string(),
|
||||
first_seen: 1700000000,
|
||||
confirmations: 5,
|
||||
zkp_proof: Vec::new(),
|
||||
},
|
||||
verified_at: 1700001000,
|
||||
stix_id: "indicator--test".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cef_format_structure() {
|
||||
let vioc = sample_vioc();
|
||||
let cef = ioc_to_cef(&vioc);
|
||||
|
||||
// CEF header has 8 pipe-delimited fields
|
||||
let parts: Vec<&str> = cef.splitn(8, '|').collect();
|
||||
assert_eq!(parts.len(), 8);
|
||||
assert_eq!(parts[0], "CEF:0");
|
||||
assert_eq!(parts[1], "Blackwall");
|
||||
assert_eq!(parts[2], "HiveMind");
|
||||
assert_eq!(parts[3], "1.0");
|
||||
assert_eq!(parts[4], "1001"); // MaliciousIp signature ID
|
||||
assert!(parts[5].contains("10.0.0.1"));
|
||||
assert_eq!(parts[6], "10"); // Critical severity
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cef_escapes_pipes() {
|
||||
assert_eq!(escape_cef_header("test|pipe"), "test\\|pipe");
|
||||
assert_eq!(escape_cef_header("back\\slash"), "back\\\\slash");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cef_escapes_extension_values() {
|
||||
assert_eq!(escape_cef_value("key=value"), "key\\=value");
|
||||
assert_eq!(escape_cef_value("line\nnew"), "line\\nnew");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cef_severity_mapping() {
|
||||
assert_eq!(cef_severity(ThreatSeverity::Info), 1);
|
||||
assert_eq!(cef_severity(ThreatSeverity::High), 8);
|
||||
assert_eq!(cef_severity(ThreatSeverity::Critical), 10);
|
||||
}
|
||||
}
|
||||
11
hivemind-api/src/integrations/mod.rs
Executable file
11
hivemind-api/src/integrations/mod.rs
Executable file
|
|
@ -0,0 +1,11 @@
|
|||
//! SIEM/SOAR integration format exporters.
|
||||
//!
|
||||
//! Converts verified IoCs to industry-standard SIEM ingestion formats:
|
||||
//!
|
||||
//! - `splunk` — Splunk HTTP Event Collector (HEC) JSON format
|
||||
//! - `qradar` — IBM QRadar LEEF (Log Event Extended Format)
|
||||
//! - `cef` — ArcSight Common Event Format (CEF)
|
||||
|
||||
pub mod cef;
|
||||
pub mod qradar;
|
||||
pub mod splunk;
|
||||
142
hivemind-api/src/integrations/qradar.rs
Executable file
142
hivemind-api/src/integrations/qradar.rs
Executable file
|
|
@ -0,0 +1,142 @@
|
|||
/// IBM QRadar LEEF (Log Event Extended Format) exporter.
|
||||
///
|
||||
/// Converts verified IoCs to LEEF 2.0 format for ingestion by
|
||||
/// IBM QRadar SIEM via log source or Syslog.
|
||||
///
|
||||
/// LEEF format: `LEEF:2.0|Vendor|Product|Version|EventID\tkey=value\tkey=value`
|
||||
///
|
||||
/// Reference: <https://www.ibm.com/docs/en/dsm?topic=leef-overview>
|
||||
use common::hivemind::{self, IoCType, ThreatSeverity};
|
||||
|
||||
use crate::store::{ip_to_string, unix_to_iso8601, VerifiedIoC};
|
||||
|
||||
/// Convert a verified IoC to LEEF 2.0 format string.
|
||||
///
|
||||
/// The returned string is a single LEEF event line suitable for
|
||||
/// Syslog forwarding or file-based ingestion.
|
||||
pub fn ioc_to_leef(vioc: &VerifiedIoC) -> String {
|
||||
let ioc = &vioc.ioc;
|
||||
let ioc_type = IoCType::from_u8(ioc.ioc_type);
|
||||
let severity = ThreatSeverity::from_u8(ioc.severity);
|
||||
|
||||
let event_id = leef_event_id(ioc_type);
|
||||
let sev = leef_severity(severity);
|
||||
|
||||
// LEEF header
|
||||
let header = format!(
|
||||
"LEEF:2.0|{}|{}|{}|{}",
|
||||
hivemind::SIEM_VENDOR,
|
||||
hivemind::SIEM_PRODUCT,
|
||||
hivemind::SIEM_VERSION,
|
||||
event_id,
|
||||
);
|
||||
|
||||
// LEEF attributes (tab-delimited)
|
||||
let src_ip = ip_to_string(ioc.ip);
|
||||
let timestamp = unix_to_iso8601(vioc.verified_at);
|
||||
let desc = escape_leef_value(&ioc.description);
|
||||
|
||||
let mut attrs = format!(
|
||||
"sev={sev}\tsrc={src_ip}\tdevTime={timestamp}\tcat={event_id}\t\
|
||||
msg={desc}\tconfirmations={}\tstix_id={}",
|
||||
ioc.confirmations,
|
||||
escape_leef_value(&vioc.stix_id),
|
||||
);
|
||||
|
||||
if let Some(ref ja4) = ioc.ja4 {
|
||||
attrs.push_str(&format!("\tja4={}", escape_leef_value(ja4)));
|
||||
}
|
||||
|
||||
if let Some(entropy) = ioc.entropy_score {
|
||||
attrs.push_str(&format!("\tentropy_score={entropy}"));
|
||||
}
|
||||
|
||||
format!("{header}\t{attrs}")
|
||||
}
|
||||
|
||||
/// Convert a batch of verified IoCs to newline-delimited LEEF.
|
||||
pub fn batch_to_leef(iocs: &[VerifiedIoC]) -> String {
|
||||
iocs.iter()
|
||||
.map(ioc_to_leef)
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n")
|
||||
}
|
||||
|
||||
/// Map IoC type to LEEF event ID.
|
||||
fn leef_event_id(t: IoCType) -> &'static str {
|
||||
match t {
|
||||
IoCType::MaliciousIp => "MaliciousIP",
|
||||
IoCType::Ja4Fingerprint => "JA4Fingerprint",
|
||||
IoCType::EntropyAnomaly => "EntropyAnomaly",
|
||||
IoCType::DnsTunnel => "DNSTunnel",
|
||||
IoCType::BehavioralPattern => "BehavioralPattern",
|
||||
}
|
||||
}
|
||||
|
||||
/// Map threat severity to LEEF numeric severity (1-10).
|
||||
fn leef_severity(s: ThreatSeverity) -> u8 {
|
||||
match s {
|
||||
ThreatSeverity::Info => 1,
|
||||
ThreatSeverity::Low => 3,
|
||||
ThreatSeverity::Medium => 5,
|
||||
ThreatSeverity::High => 7,
|
||||
ThreatSeverity::Critical => 10,
|
||||
}
|
||||
}
|
||||
|
||||
/// Escape special characters in LEEF attribute values.
|
||||
///
|
||||
/// LEEF uses tab as delimiter — tabs and newlines must be escaped.
|
||||
fn escape_leef_value(s: &str) -> String {
|
||||
s.replace('\t', "\\t")
|
||||
.replace('\n', "\\n")
|
||||
.replace('\r', "\\r")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use common::hivemind::IoC;
|
||||
|
||||
fn sample_vioc() -> VerifiedIoC {
|
||||
VerifiedIoC {
|
||||
ioc: IoC {
|
||||
ioc_type: 0,
|
||||
severity: 3,
|
||||
ip: 0xC0A80001,
|
||||
ja4: Some("t13d1516h2_abc".to_string()),
|
||||
entropy_score: Some(7500),
|
||||
description: "Malicious IP detected".to_string(),
|
||||
first_seen: 1700000000,
|
||||
confirmations: 3,
|
||||
zkp_proof: Vec::new(),
|
||||
},
|
||||
verified_at: 1700001000,
|
||||
stix_id: "indicator--aabb".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn leef_header_format() {
|
||||
let vioc = sample_vioc();
|
||||
let leef = ioc_to_leef(&vioc);
|
||||
|
||||
assert!(leef.starts_with("LEEF:2.0|Blackwall|HiveMind|1.0|MaliciousIP"));
|
||||
assert!(leef.contains("sev=7"));
|
||||
assert!(leef.contains("src=192.168.0.1"));
|
||||
assert!(leef.contains("ja4=t13d1516h2_abc"));
|
||||
assert!(leef.contains("entropy_score=7500"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn leef_escapes_special_chars() {
|
||||
let escaped = escape_leef_value("test\ttab\nnewline");
|
||||
assert_eq!(escaped, "test\\ttab\\nnewline");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn leef_severity_mapping() {
|
||||
assert_eq!(leef_severity(ThreatSeverity::Info), 1);
|
||||
assert_eq!(leef_severity(ThreatSeverity::Critical), 10);
|
||||
}
|
||||
}
|
||||
159
hivemind-api/src/integrations/splunk.rs
Executable file
159
hivemind-api/src/integrations/splunk.rs
Executable file
|
|
@ -0,0 +1,159 @@
|
|||
/// Splunk HTTP Event Collector (HEC) format exporter.
|
||||
///
|
||||
/// Converts verified IoCs to Splunk HEC JSON format suitable for
|
||||
/// direct ingestion via the Splunk HEC endpoint.
|
||||
///
|
||||
/// Format reference: <https://docs.splunk.com/Documentation/Splunk/latest/Data/FormateventsforHTTPEventCollector>
|
||||
use common::hivemind::{self, IoCType, ThreatSeverity};
|
||||
use serde::Serialize;
|
||||
|
||||
use crate::store::{ip_to_string, VerifiedIoC};
|
||||
|
||||
/// Splunk HEC event wrapper.
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
pub struct SplunkEvent {
|
||||
/// Unix timestamp of the event.
|
||||
pub time: u64,
|
||||
/// Splunk source identifier.
|
||||
pub source: &'static str,
|
||||
/// Splunk sourcetype for indexing.
|
||||
pub sourcetype: &'static str,
|
||||
/// Target Splunk index.
|
||||
pub index: &'static str,
|
||||
/// Event payload.
|
||||
pub event: SplunkEventData,
|
||||
}
|
||||
|
||||
/// Inner event data for Splunk HEC.
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
pub struct SplunkEventData {
|
||||
/// IoC type as human-readable string.
|
||||
pub ioc_type: &'static str,
|
||||
/// Severity as human-readable string.
|
||||
pub severity: &'static str,
|
||||
/// Numeric severity (0-4).
|
||||
pub severity_id: u8,
|
||||
/// Source IP in dotted notation (if applicable).
|
||||
pub src_ip: String,
|
||||
/// JA4 fingerprint (if applicable).
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub ja4: Option<String>,
|
||||
/// Byte diversity score (if applicable).
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub entropy_score: Option<u32>,
|
||||
/// Human-readable description.
|
||||
pub description: String,
|
||||
/// Number of independent confirmations.
|
||||
pub confirmations: u32,
|
||||
/// Unix timestamp when first observed.
|
||||
pub first_seen: u64,
|
||||
/// Unix timestamp when consensus was reached.
|
||||
pub verified_at: u64,
|
||||
/// STIX identifier for cross-referencing.
|
||||
pub stix_id: String,
|
||||
}
|
||||
|
||||
/// Convert a verified IoC to a Splunk HEC event.
|
||||
pub fn ioc_to_splunk(vioc: &VerifiedIoC) -> SplunkEvent {
|
||||
let ioc = &vioc.ioc;
|
||||
let ioc_type = IoCType::from_u8(ioc.ioc_type);
|
||||
let severity = ThreatSeverity::from_u8(ioc.severity);
|
||||
|
||||
SplunkEvent {
|
||||
time: vioc.verified_at,
|
||||
source: "hivemind",
|
||||
sourcetype: hivemind::SPLUNK_SOURCETYPE,
|
||||
index: "threat_intel",
|
||||
event: SplunkEventData {
|
||||
ioc_type: ioc_type_label(ioc_type),
|
||||
severity: severity_label(severity),
|
||||
severity_id: ioc.severity,
|
||||
src_ip: ip_to_string(ioc.ip),
|
||||
ja4: ioc.ja4.clone(),
|
||||
entropy_score: ioc.entropy_score,
|
||||
description: ioc.description.clone(),
|
||||
confirmations: ioc.confirmations,
|
||||
first_seen: ioc.first_seen,
|
||||
verified_at: vioc.verified_at,
|
||||
stix_id: vioc.stix_id.clone(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert a batch of verified IoCs to Splunk HEC events.
|
||||
pub fn batch_to_splunk(iocs: &[VerifiedIoC]) -> Vec<SplunkEvent> {
|
||||
iocs.iter().map(ioc_to_splunk).collect()
|
||||
}
|
||||
|
||||
/// Human-readable IoC type label.
|
||||
fn ioc_type_label(t: IoCType) -> &'static str {
|
||||
match t {
|
||||
IoCType::MaliciousIp => "malicious_ip",
|
||||
IoCType::Ja4Fingerprint => "ja4_fingerprint",
|
||||
IoCType::EntropyAnomaly => "entropy_anomaly",
|
||||
IoCType::DnsTunnel => "dns_tunnel",
|
||||
IoCType::BehavioralPattern => "behavioral_pattern",
|
||||
}
|
||||
}
|
||||
|
||||
/// Human-readable severity label.
|
||||
fn severity_label(s: ThreatSeverity) -> &'static str {
|
||||
match s {
|
||||
ThreatSeverity::Info => "info",
|
||||
ThreatSeverity::Low => "low",
|
||||
ThreatSeverity::Medium => "medium",
|
||||
ThreatSeverity::High => "high",
|
||||
ThreatSeverity::Critical => "critical",
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use common::hivemind::IoC;
|
||||
|
||||
fn sample_vioc() -> VerifiedIoC {
|
||||
VerifiedIoC {
|
||||
ioc: IoC {
|
||||
ioc_type: 0,
|
||||
severity: 3,
|
||||
ip: 0xC0A80001,
|
||||
ja4: Some("t13d1516h2_8daaf6152771_e5627efa2ab1".to_string()),
|
||||
entropy_score: Some(7500),
|
||||
description: "Malicious IP".to_string(),
|
||||
first_seen: 1700000000,
|
||||
confirmations: 3,
|
||||
zkp_proof: Vec::new(),
|
||||
},
|
||||
verified_at: 1700001000,
|
||||
stix_id: "indicator--aabbccdd-1122-3344-5566-778899aabbcc".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn splunk_event_fields() {
|
||||
let vioc = sample_vioc();
|
||||
let event = ioc_to_splunk(&vioc);
|
||||
|
||||
assert_eq!(event.time, 1700001000);
|
||||
assert_eq!(event.source, "hivemind");
|
||||
assert_eq!(event.sourcetype, hivemind::SPLUNK_SOURCETYPE);
|
||||
assert_eq!(event.event.ioc_type, "malicious_ip");
|
||||
assert_eq!(event.event.severity, "high");
|
||||
assert_eq!(event.event.src_ip, "192.168.0.1");
|
||||
assert_eq!(event.event.confirmations, 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn splunk_severity_mapping() {
|
||||
assert_eq!(severity_label(ThreatSeverity::Info), "info");
|
||||
assert_eq!(severity_label(ThreatSeverity::Critical), "critical");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn splunk_batch() {
|
||||
let iocs = vec![sample_vioc(), sample_vioc()];
|
||||
let batch = batch_to_splunk(&iocs);
|
||||
assert_eq!(batch.len(), 2);
|
||||
}
|
||||
}
|
||||
20
hivemind-api/src/lib.rs
Executable file
20
hivemind-api/src/lib.rs
Executable file
|
|
@ -0,0 +1,20 @@
|
|||
//! HiveMind Enterprise Threat Feed API.
|
||||
//!
|
||||
//! Provides REST, STIX/TAXII 2.1, and SIEM integration endpoints
|
||||
//! for consuming verified threat intelligence from the HiveMind mesh.
|
||||
//!
|
||||
//! # Modules
|
||||
//!
|
||||
//! - `store` — In-memory verified IoC storage with time-windowed queries
|
||||
//! - `stix` — STIX 2.1 types and IoC→STIX indicator conversion
|
||||
//! - `feed` — Query parameter parsing, filtering, and pagination
|
||||
//! - `integrations` — SIEM format exporters (Splunk HEC, QRadar LEEF, CEF)
|
||||
//! - `licensing` — API key management and tier-based access control
|
||||
//! - `server` — HTTP server with request routing
|
||||
|
||||
pub mod feed;
|
||||
pub mod integrations;
|
||||
pub mod licensing;
|
||||
pub mod server;
|
||||
pub mod stix;
|
||||
pub mod store;
|
||||
197
hivemind-api/src/licensing.rs
Executable file
197
hivemind-api/src/licensing.rs
Executable file
|
|
@ -0,0 +1,197 @@
|
|||
/// API key management and tier-based access control.
|
||||
///
|
||||
/// Manages API keys for the Enterprise Threat Feed. Each key is
|
||||
/// associated with an `ApiTier` that determines access to SIEM
|
||||
/// formats, page size limits, and STIX/TAXII endpoints.
|
||||
///
|
||||
/// API keys are stored as SHA256 hashes — the raw key is never
|
||||
/// persisted after initial generation.
|
||||
use common::hivemind::{self, ApiTier};
|
||||
use ring::digest;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{Arc, RwLock};
|
||||
use tracing::{info, warn};
|
||||
|
||||
/// Thread-safe handle to the license manager.
|
||||
pub type SharedLicenseManager = Arc<RwLock<LicenseManager>>;
|
||||
|
||||
/// Manages API keys and their associated tiers.
|
||||
pub struct LicenseManager {
|
||||
/// Map from SHA256(api_key) hex → tier.
|
||||
keys: HashMap<String, ApiTier>,
|
||||
}
|
||||
|
||||
impl Default for LicenseManager {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl LicenseManager {
|
||||
/// Create a new empty license manager.
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
keys: HashMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a shared (thread-safe) handle to a new license manager.
|
||||
pub fn shared() -> SharedLicenseManager {
|
||||
Arc::new(RwLock::new(Self::new()))
|
||||
}
|
||||
|
||||
/// Register an API key with the given tier.
|
||||
///
|
||||
/// The key is immediately hashed — only the hash is stored.
|
||||
/// Returns the key hash for logging purposes.
|
||||
pub fn register_key(&mut self, raw_key: &str, tier: ApiTier) -> String {
|
||||
let hash = hash_api_key(raw_key);
|
||||
self.keys.insert(hash.clone(), tier);
|
||||
info!(
|
||||
key_hash = &hash[..16],
|
||||
?tier,
|
||||
total_keys = self.keys.len(),
|
||||
"API key registered"
|
||||
);
|
||||
hash
|
||||
}
|
||||
|
||||
/// Validate an API key and return its tier.
|
||||
///
|
||||
/// Returns `None` if the key is not registered.
|
||||
pub fn validate(&self, raw_key: &str) -> Option<ApiTier> {
|
||||
let hash = hash_api_key(raw_key);
|
||||
let result = self.keys.get(&hash).copied();
|
||||
if result.is_none() {
|
||||
warn!(
|
||||
key_hash = &hash[..16],
|
||||
"API key validation failed — unknown key"
|
||||
);
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
/// Revoke an API key.
|
||||
///
|
||||
/// Returns `true` if the key existed and was removed.
|
||||
pub fn revoke_key(&mut self, raw_key: &str) -> bool {
|
||||
let hash = hash_api_key(raw_key);
|
||||
let removed = self.keys.remove(&hash).is_some();
|
||||
if removed {
|
||||
info!(key_hash = &hash[..16], "API key revoked");
|
||||
}
|
||||
removed
|
||||
}
|
||||
|
||||
/// Total number of registered API keys.
|
||||
pub fn key_count(&self) -> usize {
|
||||
self.keys.len()
|
||||
}
|
||||
}
|
||||
|
||||
/// Compute SHA256 hash of an API key, returned as lowercase hex.
|
||||
fn hash_api_key(raw_key: &str) -> String {
|
||||
let hash = digest::digest(&digest::SHA256, raw_key.as_bytes());
|
||||
hash.as_ref()
|
||||
.iter()
|
||||
.map(|b| format!("{b:02x}"))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Extract an API key from an HTTP Authorization header value.
|
||||
///
|
||||
/// Expects format: `Bearer <api-key>`
|
||||
/// Returns `None` if the header is missing or malformed.
|
||||
pub fn extract_bearer_token(auth_header: Option<&str>) -> Option<&str> {
|
||||
let header = auth_header?;
|
||||
let stripped = header.strip_prefix("Bearer ")?;
|
||||
let token = stripped.trim();
|
||||
if token.is_empty() {
|
||||
return None;
|
||||
}
|
||||
// SECURITY: Enforce maximum token length to prevent DoS via huge headers
|
||||
if token.len() > hivemind::API_KEY_LENGTH * 2 + 16 {
|
||||
return None;
|
||||
}
|
||||
Some(token)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn register_and_validate() {
|
||||
let mut mgr = LicenseManager::new();
|
||||
mgr.register_key("test-key-123", ApiTier::Enterprise);
|
||||
|
||||
let tier = mgr.validate("test-key-123");
|
||||
assert_eq!(tier, Some(ApiTier::Enterprise));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_key_returns_none() {
|
||||
let mgr = LicenseManager::new();
|
||||
assert_eq!(mgr.validate("nonexistent"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn revoke_key() {
|
||||
let mut mgr = LicenseManager::new();
|
||||
mgr.register_key("revoke-me", ApiTier::Free);
|
||||
assert!(mgr.validate("revoke-me").is_some());
|
||||
|
||||
assert!(mgr.revoke_key("revoke-me"));
|
||||
assert!(mgr.validate("revoke-me").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn different_tiers() {
|
||||
let mut mgr = LicenseManager::new();
|
||||
mgr.register_key("free-key", ApiTier::Free);
|
||||
mgr.register_key("enterprise-key", ApiTier::Enterprise);
|
||||
mgr.register_key("ns-key", ApiTier::NationalSecurity);
|
||||
|
||||
assert_eq!(mgr.validate("free-key"), Some(ApiTier::Free));
|
||||
assert_eq!(mgr.validate("enterprise-key"), Some(ApiTier::Enterprise));
|
||||
assert_eq!(
|
||||
mgr.validate("ns-key"),
|
||||
Some(ApiTier::NationalSecurity)
|
||||
);
|
||||
assert_eq!(mgr.key_count(), 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn key_hash_is_deterministic() {
|
||||
let h1 = hash_api_key("same-key");
|
||||
let h2 = hash_api_key("same-key");
|
||||
assert_eq!(h1, h2);
|
||||
assert_eq!(h1.len(), 64); // SHA256 hex = 64 chars
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_bearer_valid() {
|
||||
assert_eq!(
|
||||
extract_bearer_token(Some("Bearer my-api-key")),
|
||||
Some("my-api-key")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_bearer_missing() {
|
||||
assert_eq!(extract_bearer_token(None), None);
|
||||
assert_eq!(extract_bearer_token(Some("")), None);
|
||||
assert_eq!(extract_bearer_token(Some("Basic abc")), None);
|
||||
assert_eq!(extract_bearer_token(Some("Bearer ")), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_access_control() {
|
||||
assert!(!ApiTier::Free.can_access_siem());
|
||||
assert!(ApiTier::Enterprise.can_access_siem());
|
||||
assert!(ApiTier::NationalSecurity.can_access_taxii());
|
||||
|
||||
assert_eq!(ApiTier::Free.max_page_size(), 50);
|
||||
assert_eq!(ApiTier::Enterprise.max_page_size(), 1000);
|
||||
}
|
||||
}
|
||||
71
hivemind-api/src/main.rs
Executable file
71
hivemind-api/src/main.rs
Executable file
|
|
@ -0,0 +1,71 @@
|
|||
/// HiveMind Enterprise Threat Feed API — entry point.
|
||||
///
|
||||
/// Starts the HTTP server with configured address, initializes the
|
||||
/// in-memory IoC store and license manager, and serves threat feed
|
||||
/// endpoints.
|
||||
///
|
||||
/// # Usage
|
||||
///
|
||||
/// ```sh
|
||||
/// hivemind-api
|
||||
/// ```
|
||||
///
|
||||
/// # Configuration
|
||||
///
|
||||
/// Currently reads from defaults. Production configuration will
|
||||
/// come from a TOML file via the `common` crate config layer.
|
||||
use std::net::SocketAddr;
|
||||
|
||||
use common::hivemind;
|
||||
use tracing::info;
|
||||
use tracing_subscriber::EnvFilter;
|
||||
|
||||
use hivemind_api::licensing::LicenseManager;
|
||||
use hivemind_api::server::{self, HivemindCounters};
|
||||
use hivemind_api::store::ThreatFeedStore;
|
||||
|
||||
#[tokio::main(flavor = "current_thread")]
|
||||
async fn main() -> anyhow::Result<()> {
|
||||
// Initialize tracing
|
||||
tracing_subscriber::fmt()
|
||||
.with_env_filter(
|
||||
EnvFilter::try_from_default_env()
|
||||
.unwrap_or_else(|_| EnvFilter::new("info")),
|
||||
)
|
||||
.compact()
|
||||
.init();
|
||||
|
||||
info!(
|
||||
version = hivemind::SIEM_VERSION,
|
||||
"Starting HiveMind Enterprise Threat Feed API"
|
||||
);
|
||||
|
||||
// Initialize shared state
|
||||
let store = ThreatFeedStore::shared();
|
||||
let licensing = LicenseManager::shared();
|
||||
let counters = std::sync::Arc::new(HivemindCounters::default());
|
||||
|
||||
// Load bootstrap API keys from environment (if any)
|
||||
// SECURITY: Keys from env vars, never hardcoded
|
||||
if let Ok(key) = std::env::var("HIVEMIND_API_KEY_ENTERPRISE") {
|
||||
licensing
|
||||
.write()
|
||||
.expect("licensing lock not poisoned")
|
||||
.register_key(&key, hivemind::ApiTier::Enterprise);
|
||||
}
|
||||
if let Ok(key) = std::env::var("HIVEMIND_API_KEY_NS") {
|
||||
licensing
|
||||
.write()
|
||||
.expect("licensing lock not poisoned")
|
||||
.register_key(&key, hivemind::ApiTier::NationalSecurity);
|
||||
}
|
||||
|
||||
let addr = SocketAddr::from((
|
||||
hivemind::API_DEFAULT_ADDR
|
||||
.parse::<std::net::Ipv4Addr>()
|
||||
.expect("valid default bind address"),
|
||||
hivemind::API_DEFAULT_PORT,
|
||||
));
|
||||
|
||||
server::run(addr, store, licensing, counters).await
|
||||
}
|
||||
489
hivemind-api/src/server.rs
Executable file
489
hivemind-api/src/server.rs
Executable file
|
|
@ -0,0 +1,489 @@
|
|||
/// HTTP server with request routing and response formatting.
|
||||
///
|
||||
/// Implements a hyper 1.x HTTP server with manual path-based routing.
|
||||
/// All endpoints require API key authentication via `Authorization: Bearer <key>`
|
||||
/// header, except for the TAXII discovery endpoint.
|
||||
///
|
||||
/// # Endpoints
|
||||
///
|
||||
/// | Path | Description | Tier |
|
||||
/// |------|-------------|------|
|
||||
/// | `GET /taxii2/` | TAXII 2.1 API root discovery | Any |
|
||||
/// | `GET /taxii2/collections/` | List TAXII collections | Enterprise+ |
|
||||
/// | `GET /taxii2/collections/{id}/objects/` | STIX objects | Enterprise+ |
|
||||
/// | `GET /api/v1/feed` | JSON feed of verified IoCs | Any |
|
||||
/// | `GET /api/v1/feed/stix` | STIX 2.1 bundle | Enterprise+ |
|
||||
/// | `GET /api/v1/feed/splunk` | Splunk HEC format | Enterprise+ |
|
||||
/// | `GET /api/v1/feed/qradar` | QRadar LEEF format | Enterprise+ |
|
||||
/// | `GET /api/v1/feed/cef` | CEF format | Enterprise+ |
|
||||
/// | `GET /api/v1/stats` | Feed statistics | Any |
|
||||
use std::convert::Infallible;
|
||||
use std::net::SocketAddr;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::sync::Arc;
|
||||
|
||||
use common::hivemind::{self, ApiTier};
|
||||
use http_body_util::{BodyExt, Full};
|
||||
use hyper::body::Bytes;
|
||||
use hyper::server::conn::http1;
|
||||
use hyper::service::service_fn;
|
||||
use hyper::{Method, Request, Response, StatusCode};
|
||||
use hyper_util::rt::TokioIo;
|
||||
use tokio::net::TcpListener;
|
||||
use tracing::{error, info, warn};
|
||||
|
||||
use crate::feed;
|
||||
use crate::integrations::{cef, qradar, splunk};
|
||||
use crate::licensing::{self, SharedLicenseManager};
|
||||
use crate::stix;
|
||||
use crate::store::SharedStore;
|
||||
|
||||
/// Live counters pushed by blackwall, hivemind, and enterprise daemons (optional)
|
||||
/// via `POST /push`. Each daemon pushes only its own fields.
|
||||
#[derive(Default)]
|
||||
pub struct HivemindCounters {
|
||||
// eBPF/XDP counters (pushed by blackwall daemon)
|
||||
pub packets_total: AtomicU64,
|
||||
pub packets_passed: AtomicU64,
|
||||
pub packets_dropped: AtomicU64,
|
||||
pub anomalies_sent: AtomicU64,
|
||||
|
||||
// P2P mesh counters (pushed by hivemind daemon)
|
||||
pub peer_count: AtomicU64,
|
||||
pub iocs_shared_p2p: AtomicU64,
|
||||
pub avg_reputation_x100: AtomicU64,
|
||||
pub messages_total: AtomicU64,
|
||||
|
||||
// A2A counters (pushed by enterprise module when active)
|
||||
pub a2a_jwts_verified: AtomicU64,
|
||||
pub a2a_violations: AtomicU64,
|
||||
pub a2a_injections: AtomicU64,
|
||||
}
|
||||
|
||||
pub type SharedCounters = Arc<HivemindCounters>;
|
||||
|
||||
/// Delta payload for `POST /push`.
|
||||
///
|
||||
/// All fields are optional so each daemon can push only its own counters
|
||||
/// without zeroing out counters owned by other daemons.
|
||||
#[derive(serde::Deserialize)]
|
||||
struct CounterDelta {
|
||||
// eBPF counters (from blackwall)
|
||||
packets_total: Option<u64>,
|
||||
packets_passed: Option<u64>,
|
||||
packets_dropped: Option<u64>,
|
||||
anomalies_sent: Option<u64>,
|
||||
|
||||
// P2P counters (from hivemind)
|
||||
peer_count: Option<u64>,
|
||||
iocs_shared_p2p: Option<u64>,
|
||||
avg_reputation_x100: Option<u64>,
|
||||
messages_total: Option<u64>,
|
||||
|
||||
// A2A counters (from enterprise module)
|
||||
a2a_jwts_verified: Option<u64>,
|
||||
a2a_violations: Option<u64>,
|
||||
a2a_injections: Option<u64>,
|
||||
}
|
||||
|
||||
/// Start the HTTP server and listen for connections.
|
||||
///
|
||||
/// This function runs forever (until the process is terminated).
|
||||
/// Each incoming connection spawns a new task for HTTP/1.1 handling.
|
||||
pub async fn run(
|
||||
addr: SocketAddr,
|
||||
store: SharedStore,
|
||||
licensing: SharedLicenseManager,
|
||||
counters: SharedCounters,
|
||||
) -> anyhow::Result<()> {
|
||||
let listener = TcpListener::bind(addr).await?;
|
||||
info!(%addr, "Enterprise Threat Feed API listening");
|
||||
|
||||
loop {
|
||||
let (stream, peer) = listener.accept().await?;
|
||||
let io = TokioIo::new(stream);
|
||||
let store = store.clone();
|
||||
let licensing = licensing.clone();
|
||||
let counters = counters.clone();
|
||||
|
||||
tokio::task::spawn(async move {
|
||||
let service = service_fn(move |req| {
|
||||
let store = store.clone();
|
||||
let licensing = licensing.clone();
|
||||
let counters = counters.clone();
|
||||
async move { handle_request(req, store, licensing, counters, peer).await }
|
||||
});
|
||||
|
||||
if let Err(e) = http1::Builder::new()
|
||||
.serve_connection(io, service)
|
||||
.await
|
||||
{
|
||||
error!(peer = %peer, error = %e, "HTTP connection error");
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/// Route an HTTP request to the appropriate handler.
|
||||
async fn handle_request(
|
||||
req: Request<hyper::body::Incoming>,
|
||||
store: SharedStore,
|
||||
licensing: SharedLicenseManager,
|
||||
counters: SharedCounters,
|
||||
peer: SocketAddr,
|
||||
) -> Result<Response<Full<Bytes>>, Infallible> {
|
||||
let method = req.method().clone();
|
||||
let path = req.uri().path().to_string();
|
||||
let query = req.uri().query().map(|q| q.to_string());
|
||||
|
||||
// Extract API key
|
||||
let auth_header = req
|
||||
.headers()
|
||||
.get("authorization")
|
||||
.and_then(|v| v.to_str().ok());
|
||||
let token = licensing::extract_bearer_token(auth_header);
|
||||
let had_token = token.is_some();
|
||||
|
||||
// Validate API key
|
||||
let tier = match token {
|
||||
Some(key) => {
|
||||
let mgr = licensing
|
||||
.read()
|
||||
.expect("licensing lock not poisoned");
|
||||
mgr.validate(key)
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
|
||||
info!(
|
||||
%peer,
|
||||
%method,
|
||||
path = %path,
|
||||
authenticated = tier.is_some(),
|
||||
"Request received"
|
||||
);
|
||||
|
||||
// Route based on method + path
|
||||
let response = match (&method, path.as_str()) {
|
||||
// TAXII 2.1 endpoints
|
||||
(&Method::GET, "/taxii2/") => handle_taxii_discovery(),
|
||||
|
||||
(&Method::GET, "/taxii2/collections/") => {
|
||||
require_taxii(tier, had_token, handle_taxii_collections)
|
||||
}
|
||||
|
||||
(&Method::GET, p) if is_taxii_objects_path(p) => {
|
||||
require_taxii(tier, had_token, || {
|
||||
let store = store
|
||||
.read()
|
||||
.expect("store lock not poisoned");
|
||||
let max_page = tier.map_or(50, |t| t.max_page_size());
|
||||
let params = feed::parse_query_params(query.as_deref(), max_page);
|
||||
let result = store.query(¶ms);
|
||||
let bundle = stix::build_bundle(&result.items);
|
||||
json_response(StatusCode::OK, hivemind::STIX_CONTENT_TYPE, &bundle)
|
||||
})
|
||||
}
|
||||
|
||||
// Custom REST endpoints
|
||||
(&Method::GET, "/api/v1/feed") => {
|
||||
let effective_tier = tier.unwrap_or(ApiTier::Free);
|
||||
let store = store
|
||||
.read()
|
||||
.expect("store lock not poisoned");
|
||||
let params = feed::parse_query_params(
|
||||
query.as_deref(),
|
||||
effective_tier.max_page_size(),
|
||||
);
|
||||
let result = store.query(¶ms);
|
||||
json_response(StatusCode::OK, "application/json", &result)
|
||||
}
|
||||
|
||||
(&Method::GET, "/api/v1/feed/stix") => {
|
||||
require_taxii(tier, had_token, || {
|
||||
let store = store
|
||||
.read()
|
||||
.expect("store lock not poisoned");
|
||||
let max_page = tier.map_or(50, |t| t.max_page_size());
|
||||
let params = feed::parse_query_params(query.as_deref(), max_page);
|
||||
let result = store.query(¶ms);
|
||||
let bundle = stix::build_bundle(&result.items);
|
||||
json_response(StatusCode::OK, hivemind::STIX_CONTENT_TYPE, &bundle)
|
||||
})
|
||||
}
|
||||
|
||||
(&Method::GET, "/api/v1/feed/splunk") => {
|
||||
require_siem(tier, had_token, || {
|
||||
let store = store
|
||||
.read()
|
||||
.expect("store lock not poisoned");
|
||||
let max_page = tier.map_or(50, |t| t.max_page_size());
|
||||
let params = feed::parse_query_params(query.as_deref(), max_page);
|
||||
let result = store.query(¶ms);
|
||||
let events = splunk::batch_to_splunk(&result.items);
|
||||
json_response(StatusCode::OK, "application/json", &events)
|
||||
})
|
||||
}
|
||||
|
||||
(&Method::GET, "/api/v1/feed/qradar") => {
|
||||
require_siem(tier, had_token, || {
|
||||
let store = store
|
||||
.read()
|
||||
.expect("store lock not poisoned");
|
||||
let max_page = tier.map_or(50, |t| t.max_page_size());
|
||||
let params = feed::parse_query_params(query.as_deref(), max_page);
|
||||
let result = store.query(¶ms);
|
||||
text_response(
|
||||
StatusCode::OK,
|
||||
"text/plain",
|
||||
&qradar::batch_to_leef(&result.items),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
(&Method::GET, "/api/v1/feed/cef") => {
|
||||
require_siem(tier, had_token, || {
|
||||
let store = store
|
||||
.read()
|
||||
.expect("store lock not poisoned");
|
||||
let max_page = tier.map_or(50, |t| t.max_page_size());
|
||||
let params = feed::parse_query_params(query.as_deref(), max_page);
|
||||
let result = store.query(¶ms);
|
||||
text_response(
|
||||
StatusCode::OK,
|
||||
"text/plain",
|
||||
&cef::batch_to_cef(&result.items),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
(&Method::GET, "/api/v1/stats") => {
|
||||
let store = store
|
||||
.read()
|
||||
.expect("store lock not poisoned");
|
||||
let stats = feed::compute_stats(&store);
|
||||
json_response(StatusCode::OK, "application/json", &stats)
|
||||
}
|
||||
|
||||
// Dashboard mesh stats endpoint (no auth required)
|
||||
(&Method::GET, "/stats") => {
|
||||
let store = store
|
||||
.read()
|
||||
.expect("store lock not poisoned");
|
||||
let mesh = feed::compute_mesh_stats(&store, &counters);
|
||||
json_response(StatusCode::OK, "application/json", &mesh)
|
||||
}
|
||||
|
||||
// Internal metrics push from blackwall daemon (localhost only)
|
||||
(&Method::POST, "/push") => {
|
||||
// SECURITY: only accept from loopback
|
||||
if !peer.ip().is_loopback() {
|
||||
warn!(%peer, "rejected /push from non-loopback");
|
||||
return Ok(error_response(StatusCode::FORBIDDEN, "Forbidden"));
|
||||
}
|
||||
let body_bytes = match req.collect().await {
|
||||
Ok(b) => b.to_bytes(),
|
||||
Err(_) => return Ok(error_response(StatusCode::BAD_REQUEST, "bad body")),
|
||||
};
|
||||
match serde_json::from_slice::<CounterDelta>(&body_bytes) {
|
||||
Ok(delta) => {
|
||||
// eBPF counters (from blackwall)
|
||||
if let Some(v) = delta.packets_total {
|
||||
counters.packets_total.store(v, Ordering::Relaxed);
|
||||
}
|
||||
if let Some(v) = delta.packets_passed {
|
||||
counters.packets_passed.store(v, Ordering::Relaxed);
|
||||
}
|
||||
if let Some(v) = delta.packets_dropped {
|
||||
counters.packets_dropped.store(v, Ordering::Relaxed);
|
||||
}
|
||||
if let Some(v) = delta.anomalies_sent {
|
||||
counters.anomalies_sent.store(v, Ordering::Relaxed);
|
||||
}
|
||||
// P2P counters (from hivemind)
|
||||
if let Some(v) = delta.peer_count {
|
||||
counters.peer_count.store(v, Ordering::Relaxed);
|
||||
}
|
||||
if let Some(v) = delta.iocs_shared_p2p {
|
||||
counters.iocs_shared_p2p.store(v, Ordering::Relaxed);
|
||||
}
|
||||
if let Some(v) = delta.avg_reputation_x100 {
|
||||
counters.avg_reputation_x100.store(v, Ordering::Relaxed);
|
||||
}
|
||||
if let Some(v) = delta.messages_total {
|
||||
counters.messages_total.store(v, Ordering::Relaxed);
|
||||
}
|
||||
// A2A counters (from enterprise module)
|
||||
if let Some(v) = delta.a2a_jwts_verified {
|
||||
counters.a2a_jwts_verified.store(v, Ordering::Relaxed);
|
||||
}
|
||||
if let Some(v) = delta.a2a_violations {
|
||||
counters.a2a_violations.store(v, Ordering::Relaxed);
|
||||
}
|
||||
if let Some(v) = delta.a2a_injections {
|
||||
counters.a2a_injections.store(v, Ordering::Relaxed);
|
||||
}
|
||||
json_response(StatusCode::OK, "application/json", &serde_json::json!({"ok": true}))
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(%e, "failed to parse /push payload");
|
||||
error_response(StatusCode::BAD_REQUEST, "invalid JSON")
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
warn!(%method, path = %path, "Unknown endpoint");
|
||||
error_response(StatusCode::NOT_FOUND, "Not found")
|
||||
}
|
||||
};
|
||||
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
// --- TAXII endpoint handlers ---
|
||||
|
||||
/// Handle TAXII 2.1 API root discovery (no auth required).
|
||||
fn handle_taxii_discovery() -> Response<Full<Bytes>> {
|
||||
let discovery = stix::discovery_response();
|
||||
json_response(StatusCode::OK, hivemind::TAXII_CONTENT_TYPE, &discovery)
|
||||
}
|
||||
|
||||
/// Handle TAXII 2.1 collection listing.
|
||||
fn handle_taxii_collections() -> Response<Full<Bytes>> {
|
||||
let collections = vec![stix::default_collection()];
|
||||
let wrapper = serde_json::json!({ "collections": collections });
|
||||
json_response(StatusCode::OK, hivemind::TAXII_CONTENT_TYPE, &wrapper)
|
||||
}
|
||||
|
||||
// --- Access control helpers ---
|
||||
|
||||
/// Require Enterprise+ tier for TAXII endpoints.
|
||||
fn require_taxii<F>(tier: Option<ApiTier>, had_token: bool, f: F) -> Response<Full<Bytes>>
|
||||
where
|
||||
F: FnOnce() -> Response<Full<Bytes>>,
|
||||
{
|
||||
match tier {
|
||||
Some(t) if t.can_access_taxii() => f(),
|
||||
Some(_) => error_response(
|
||||
StatusCode::FORBIDDEN,
|
||||
"TAXII endpoints require Enterprise or NationalSecurity tier",
|
||||
),
|
||||
None if had_token => error_response(
|
||||
StatusCode::UNAUTHORIZED,
|
||||
"Invalid API key",
|
||||
),
|
||||
None => error_response(
|
||||
StatusCode::UNAUTHORIZED,
|
||||
"Authorization header with Bearer token required",
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
/// Require Enterprise+ tier for SIEM integration endpoints.
|
||||
fn require_siem<F>(tier: Option<ApiTier>, had_token: bool, f: F) -> Response<Full<Bytes>>
|
||||
where
|
||||
F: FnOnce() -> Response<Full<Bytes>>,
|
||||
{
|
||||
match tier {
|
||||
Some(t) if t.can_access_siem() => f(),
|
||||
Some(_) => error_response(
|
||||
StatusCode::FORBIDDEN,
|
||||
"SIEM integration endpoints require Enterprise or NationalSecurity tier",
|
||||
),
|
||||
None if had_token => error_response(
|
||||
StatusCode::UNAUTHORIZED,
|
||||
"Invalid API key",
|
||||
),
|
||||
None => error_response(
|
||||
StatusCode::UNAUTHORIZED,
|
||||
"Authorization header with Bearer token required",
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
// --- Path matching ---
|
||||
|
||||
/// Check if a path matches the TAXII collection objects pattern.
|
||||
///
|
||||
/// Pattern: `/taxii2/collections/<id>/objects/`
|
||||
fn is_taxii_objects_path(path: &str) -> bool {
|
||||
let Some(rest) = path.strip_prefix("/taxii2/collections/") else {
|
||||
return false;
|
||||
};
|
||||
rest.ends_with("/objects/") && rest.len() > "/objects/".len()
|
||||
}
|
||||
|
||||
// --- Response builders ---
|
||||
|
||||
/// Build a JSON response with the given status and content type.
|
||||
fn json_response<T: serde::Serialize>(
|
||||
status: StatusCode,
|
||||
content_type: &str,
|
||||
body: &T,
|
||||
) -> Response<Full<Bytes>> {
|
||||
let json = serde_json::to_string(body).unwrap_or_else(|e| {
|
||||
format!("{{\"error\":\"serialization failed: {e}\"}}")
|
||||
});
|
||||
Response::builder()
|
||||
.status(status)
|
||||
.header("content-type", content_type)
|
||||
.header("x-hivemind-version", hivemind::SIEM_VERSION)
|
||||
.body(Full::new(Bytes::from(json)))
|
||||
.expect("building response with valid parameters")
|
||||
}
|
||||
|
||||
/// Build a plain-text response.
|
||||
fn text_response(
|
||||
status: StatusCode,
|
||||
content_type: &str,
|
||||
body: &str,
|
||||
) -> Response<Full<Bytes>> {
|
||||
Response::builder()
|
||||
.status(status)
|
||||
.header("content-type", content_type)
|
||||
.header("x-hivemind-version", hivemind::SIEM_VERSION)
|
||||
.body(Full::new(Bytes::from(body.to_owned())))
|
||||
.expect("building response with valid parameters")
|
||||
}
|
||||
|
||||
/// Build a JSON error response.
|
||||
fn error_response(status: StatusCode, message: &str) -> Response<Full<Bytes>> {
|
||||
let body = serde_json::json!({
|
||||
"error": message,
|
||||
"status": status.as_u16(),
|
||||
});
|
||||
json_response(status, "application/json", &body)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn taxii_objects_path_matching() {
|
||||
assert!(is_taxii_objects_path(
|
||||
"/taxii2/collections/hivemind-threat-feed-v1/objects/"
|
||||
));
|
||||
assert!(!is_taxii_objects_path("/taxii2/collections/"));
|
||||
assert!(!is_taxii_objects_path("/taxii2/collections/objects/"));
|
||||
assert!(!is_taxii_objects_path("/api/v1/feed"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn error_response_format() {
|
||||
let resp = error_response(StatusCode::UNAUTHORIZED, "test error");
|
||||
assert_eq!(resp.status(), StatusCode::UNAUTHORIZED);
|
||||
let ct = resp.headers().get("content-type").expect("has content-type");
|
||||
assert_eq!(ct, "application/json");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn json_response_has_version_header() {
|
||||
let resp = json_response(StatusCode::OK, "application/json", &"hello");
|
||||
let ver = resp
|
||||
.headers()
|
||||
.get("x-hivemind-version")
|
||||
.expect("has version");
|
||||
assert_eq!(ver, hivemind::SIEM_VERSION);
|
||||
}
|
||||
}
|
||||
328
hivemind-api/src/stix.rs
Executable file
328
hivemind-api/src/stix.rs
Executable file
|
|
@ -0,0 +1,328 @@
|
|||
/// STIX 2.1 types and IoC-to-STIX conversion.
|
||||
///
|
||||
/// Implements core STIX Structured Threat Information Expression objects
|
||||
/// for the Enterprise Threat Feed API. Converts HiveMind IoCs to
|
||||
/// STIX Indicator SDOs within STIX Bundles.
|
||||
///
|
||||
/// Reference: <https://docs.oasis-open.org/cti/stix/v2.1/os/stix-v2.1-os.html>
|
||||
use common::hivemind::{self, IoCType, ThreatSeverity};
|
||||
use serde::Serialize;
|
||||
|
||||
use crate::store::{ip_to_string, unix_to_iso8601, VerifiedIoC};
|
||||
|
||||
/// STIX 2.1 Bundle — top-level container for STIX objects.
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
pub struct StixBundle {
|
||||
/// Always "bundle".
|
||||
#[serde(rename = "type")]
|
||||
pub object_type: &'static str,
|
||||
/// Deterministic bundle ID.
|
||||
pub id: String,
|
||||
/// List of STIX objects.
|
||||
pub objects: Vec<StixIndicator>,
|
||||
}
|
||||
|
||||
/// STIX 2.1 Indicator SDO — represents an IoC observation.
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
pub struct StixIndicator {
|
||||
/// Always "indicator".
|
||||
#[serde(rename = "type")]
|
||||
pub object_type: &'static str,
|
||||
/// STIX spec version.
|
||||
pub spec_version: &'static str,
|
||||
/// Deterministic STIX ID (from store).
|
||||
pub id: String,
|
||||
/// ISO 8601 creation timestamp.
|
||||
pub created: String,
|
||||
/// ISO 8601 modification timestamp.
|
||||
pub modified: String,
|
||||
/// Human-readable indicator name.
|
||||
pub name: String,
|
||||
/// STIX pattern expression.
|
||||
pub pattern: String,
|
||||
/// Pattern language (always "stix").
|
||||
pub pattern_type: &'static str,
|
||||
/// When this indicator becomes valid.
|
||||
pub valid_from: String,
|
||||
/// Confidence score (0-100).
|
||||
pub confidence: u8,
|
||||
/// Indicator type labels.
|
||||
pub indicator_types: Vec<&'static str>,
|
||||
/// Descriptive labels.
|
||||
pub labels: Vec<String>,
|
||||
}
|
||||
|
||||
/// TAXII 2.1 Collection resource.
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
pub struct TaxiiCollection {
|
||||
/// Collection identifier.
|
||||
pub id: String,
|
||||
/// Human-readable title.
|
||||
pub title: String,
|
||||
/// Description.
|
||||
pub description: String,
|
||||
/// Whether this collection can be read.
|
||||
pub can_read: bool,
|
||||
/// Whether this collection can be written to.
|
||||
pub can_write: bool,
|
||||
/// Supported media types.
|
||||
pub media_types: Vec<&'static str>,
|
||||
}
|
||||
|
||||
/// TAXII 2.1 API Root discovery response.
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
pub struct TaxiiDiscovery {
|
||||
/// API root title.
|
||||
pub title: String,
|
||||
/// Description.
|
||||
pub description: String,
|
||||
/// Supported TAXII versions.
|
||||
pub versions: Vec<&'static str>,
|
||||
/// Maximum content length.
|
||||
pub max_content_length: usize,
|
||||
}
|
||||
|
||||
/// Convert a verified IoC to a STIX 2.1 Indicator.
|
||||
pub fn ioc_to_indicator(vioc: &VerifiedIoC) -> StixIndicator {
|
||||
let ioc = &vioc.ioc;
|
||||
let ioc_type = IoCType::from_u8(ioc.ioc_type);
|
||||
let severity = ThreatSeverity::from_u8(ioc.severity);
|
||||
|
||||
let name = build_indicator_name(ioc_type, ioc);
|
||||
let pattern = build_stix_pattern(ioc_type, ioc);
|
||||
let confidence = severity_to_confidence(severity);
|
||||
let indicator_types = ioc_type_to_stix_types(ioc_type);
|
||||
let created = unix_to_iso8601(vioc.verified_at);
|
||||
let valid_from = unix_to_iso8601(ioc.first_seen);
|
||||
|
||||
StixIndicator {
|
||||
object_type: "indicator",
|
||||
spec_version: hivemind::STIX_SPEC_VERSION,
|
||||
id: vioc.stix_id.clone(),
|
||||
created: created.clone(),
|
||||
modified: created,
|
||||
name,
|
||||
pattern,
|
||||
pattern_type: "stix",
|
||||
valid_from,
|
||||
confidence,
|
||||
indicator_types,
|
||||
labels: vec![format!("severity:{}", ioc.severity)],
|
||||
}
|
||||
}
|
||||
|
||||
/// Build a STIX bundle from a list of verified IoCs.
|
||||
pub fn build_bundle(iocs: &[VerifiedIoC]) -> StixBundle {
|
||||
let objects: Vec<StixIndicator> = iocs.iter().map(ioc_to_indicator).collect();
|
||||
|
||||
// Bundle ID: deterministic from object count + first ID
|
||||
let bundle_suffix = if let Some(first) = objects.first() {
|
||||
first.id.chars().skip(12).take(36).collect::<String>()
|
||||
} else {
|
||||
"00000000-0000-0000-0000-000000000000".to_string()
|
||||
};
|
||||
|
||||
StixBundle {
|
||||
object_type: "bundle",
|
||||
id: format!("bundle--{bundle_suffix}"),
|
||||
objects,
|
||||
}
|
||||
}
|
||||
|
||||
/// Build the default TAXII collection descriptor.
|
||||
pub fn default_collection() -> TaxiiCollection {
|
||||
TaxiiCollection {
|
||||
id: hivemind::TAXII_COLLECTION_ID.to_string(),
|
||||
title: hivemind::TAXII_COLLECTION_TITLE.to_string(),
|
||||
description: "Consensus-verified threat indicators from the HiveMind P2P mesh. \
|
||||
Each IoC has been cross-validated by at least 3 independent peers."
|
||||
.to_string(),
|
||||
can_read: true,
|
||||
can_write: false,
|
||||
media_types: vec![hivemind::STIX_CONTENT_TYPE],
|
||||
}
|
||||
}
|
||||
|
||||
/// Build the TAXII API root discovery response.
|
||||
pub fn discovery_response() -> TaxiiDiscovery {
|
||||
TaxiiDiscovery {
|
||||
title: "HiveMind Threat Feed".to_string(),
|
||||
description: "TAXII 2.1 API for the HiveMind decentralized threat intelligence mesh."
|
||||
.to_string(),
|
||||
versions: vec!["taxii-2.1"],
|
||||
max_content_length: hivemind::MAX_MESSAGE_SIZE,
|
||||
}
|
||||
}
|
||||
|
||||
/// Build a human-readable indicator name from IoC fields.
|
||||
fn build_indicator_name(ioc_type: IoCType, ioc: &common::hivemind::IoC) -> String {
|
||||
match ioc_type {
|
||||
IoCType::MaliciousIp => {
|
||||
format!("Malicious IP {}", ip_to_string(ioc.ip))
|
||||
}
|
||||
IoCType::Ja4Fingerprint => {
|
||||
let ja4 = ioc.ja4.as_deref().unwrap_or("unknown");
|
||||
format!("Malicious JA4 fingerprint {ja4}")
|
||||
}
|
||||
IoCType::EntropyAnomaly => {
|
||||
let score = ioc.entropy_score.unwrap_or(0);
|
||||
format!("High-entropy anomaly (score={score}) from {}", ip_to_string(ioc.ip))
|
||||
}
|
||||
IoCType::DnsTunnel => {
|
||||
format!("DNS tunneling from {}", ip_to_string(ioc.ip))
|
||||
}
|
||||
IoCType::BehavioralPattern => {
|
||||
format!("Behavioral anomaly from {}", ip_to_string(ioc.ip))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Build a STIX pattern expression from an IoC.
|
||||
///
|
||||
/// STIX patterns follow the STIX Patterning language:
|
||||
/// `[<object-type>:<property> = '<value>']`
|
||||
fn build_stix_pattern(ioc_type: IoCType, ioc: &common::hivemind::IoC) -> String {
|
||||
match ioc_type {
|
||||
IoCType::MaliciousIp => {
|
||||
format!("[ipv4-addr:value = '{}']", ip_to_string(ioc.ip))
|
||||
}
|
||||
IoCType::Ja4Fingerprint => {
|
||||
let ja4 = ioc.ja4.as_deref().unwrap_or("unknown");
|
||||
format!("[network-traffic:extensions.'tls-ext'.ja4 = '{ja4}']")
|
||||
}
|
||||
IoCType::EntropyAnomaly => {
|
||||
format!(
|
||||
"[network-traffic:src_ref.type = 'ipv4-addr' AND \
|
||||
network-traffic:src_ref.value = '{}']",
|
||||
ip_to_string(ioc.ip)
|
||||
)
|
||||
}
|
||||
IoCType::DnsTunnel => {
|
||||
format!(
|
||||
"[domain-name:resolves_to_refs[*].value = '{}']",
|
||||
ip_to_string(ioc.ip)
|
||||
)
|
||||
}
|
||||
IoCType::BehavioralPattern => {
|
||||
format!(
|
||||
"[network-traffic:src_ref.type = 'ipv4-addr' AND \
|
||||
network-traffic:src_ref.value = '{}']",
|
||||
ip_to_string(ioc.ip)
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Map threat severity to STIX confidence score (0-100).
|
||||
fn severity_to_confidence(severity: ThreatSeverity) -> u8 {
|
||||
match severity {
|
||||
ThreatSeverity::Info => 20,
|
||||
ThreatSeverity::Low => 40,
|
||||
ThreatSeverity::Medium => 60,
|
||||
ThreatSeverity::High => 80,
|
||||
ThreatSeverity::Critical => 95,
|
||||
}
|
||||
}
|
||||
|
||||
/// Map IoC type to STIX indicator type labels.
|
||||
fn ioc_type_to_stix_types(ioc_type: IoCType) -> Vec<&'static str> {
|
||||
match ioc_type {
|
||||
IoCType::MaliciousIp => vec!["malicious-activity", "anomalous-activity"],
|
||||
IoCType::Ja4Fingerprint => vec!["malicious-activity"],
|
||||
IoCType::EntropyAnomaly => vec!["anomalous-activity"],
|
||||
IoCType::DnsTunnel => vec!["malicious-activity", "anomalous-activity"],
|
||||
IoCType::BehavioralPattern => vec!["anomalous-activity"],
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use common::hivemind::IoC;
|
||||
|
||||
fn sample_ioc() -> IoC {
|
||||
IoC {
|
||||
ioc_type: 0, // MaliciousIp
|
||||
severity: 3, // High
|
||||
ip: 0xC0A80001,
|
||||
ja4: Some("t13d1516h2_8daaf6152771_e5627efa2ab1".to_string()),
|
||||
entropy_score: Some(7500),
|
||||
description: "Test malicious IP".to_string(),
|
||||
first_seen: 1700000000,
|
||||
confirmations: 3,
|
||||
zkp_proof: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn sample_verified() -> VerifiedIoC {
|
||||
VerifiedIoC {
|
||||
stix_id: "indicator--aabbccdd-1122-3344-5566-778899aabbcc".to_string(),
|
||||
verified_at: 1700001000,
|
||||
ioc: sample_ioc(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ioc_converts_to_indicator() {
|
||||
let vioc = sample_verified();
|
||||
let indicator = ioc_to_indicator(&vioc);
|
||||
|
||||
assert_eq!(indicator.object_type, "indicator");
|
||||
assert_eq!(indicator.spec_version, "2.1");
|
||||
assert_eq!(indicator.id, vioc.stix_id);
|
||||
assert_eq!(indicator.pattern_type, "stix");
|
||||
assert_eq!(indicator.confidence, 80); // High severity
|
||||
assert!(indicator.name.contains("192.168.0.1"));
|
||||
assert!(indicator.pattern.contains("192.168.0.1"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bundle_creation() {
|
||||
let viocs = vec![sample_verified()];
|
||||
let bundle = build_bundle(&viocs);
|
||||
|
||||
assert_eq!(bundle.object_type, "bundle");
|
||||
assert!(bundle.id.starts_with("bundle--"));
|
||||
assert_eq!(bundle.objects.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_bundle() {
|
||||
let bundle = build_bundle(&[]);
|
||||
assert_eq!(bundle.objects.len(), 0);
|
||||
assert!(bundle.id.starts_with("bundle--"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stix_patterns_by_type() {
|
||||
let ioc = sample_ioc();
|
||||
|
||||
// MaliciousIp
|
||||
let pattern = build_stix_pattern(IoCType::MaliciousIp, &ioc);
|
||||
assert!(pattern.starts_with("[ipv4-addr:value"));
|
||||
|
||||
// Ja4Fingerprint
|
||||
let mut ja4_ioc = ioc.clone();
|
||||
ja4_ioc.ioc_type = 1;
|
||||
let pattern = build_stix_pattern(IoCType::Ja4Fingerprint, &ja4_ioc);
|
||||
assert!(pattern.contains("tls-ext"));
|
||||
|
||||
// DnsTunnel
|
||||
let pattern = build_stix_pattern(IoCType::DnsTunnel, &ioc);
|
||||
assert!(pattern.contains("domain-name"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn taxii_discovery() {
|
||||
let disc = discovery_response();
|
||||
assert!(disc.versions.contains(&"taxii-2.1"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn taxii_collection() {
|
||||
let coll = default_collection();
|
||||
assert_eq!(coll.id, hivemind::TAXII_COLLECTION_ID);
|
||||
assert!(coll.can_read);
|
||||
assert!(!coll.can_write);
|
||||
}
|
||||
}
|
||||
351
hivemind-api/src/store.rs
Executable file
351
hivemind-api/src/store.rs
Executable file
|
|
@ -0,0 +1,351 @@
|
|||
/// In-memory store for consensus-verified IoCs.
|
||||
///
|
||||
/// The `ThreatFeedStore` holds all IoCs that reached cross-validation
|
||||
/// consensus in the HiveMind mesh. It supports time-windowed queries,
|
||||
/// filtering by severity and type, and pagination for API responses.
|
||||
use common::hivemind::{self, IoC};
|
||||
use ring::digest;
|
||||
use std::sync::{Arc, RwLock};
|
||||
use tracing::info;
|
||||
|
||||
/// A consensus-verified IoC with feed metadata.
|
||||
#[derive(Clone, Debug, serde::Serialize)]
|
||||
pub struct VerifiedIoC {
|
||||
/// The verified IoC data.
|
||||
pub ioc: IoC,
|
||||
/// Unix timestamp when consensus was reached.
|
||||
pub verified_at: u64,
|
||||
/// Pre-computed deterministic STIX identifier.
|
||||
pub stix_id: String,
|
||||
}
|
||||
|
||||
/// Thread-safe handle to the IoC store.
|
||||
pub type SharedStore = Arc<RwLock<ThreatFeedStore>>;
|
||||
|
||||
/// In-memory storage for verified IoCs, sorted by verification time.
|
||||
pub struct ThreatFeedStore {
|
||||
/// Verified IoCs, ordered by `verified_at` ascending.
|
||||
iocs: Vec<VerifiedIoC>,
|
||||
}
|
||||
|
||||
impl Default for ThreatFeedStore {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl ThreatFeedStore {
|
||||
/// Create a new empty store.
|
||||
pub fn new() -> Self {
|
||||
Self { iocs: Vec::new() }
|
||||
}
|
||||
|
||||
/// Create a shared (thread-safe) handle to a new store.
|
||||
pub fn shared() -> SharedStore {
|
||||
Arc::new(RwLock::new(Self::new()))
|
||||
}
|
||||
|
||||
/// Insert a verified IoC into the store.
|
||||
///
|
||||
/// Computes the deterministic STIX ID from the IoC fields and
|
||||
/// inserts in sorted order by verification timestamp.
|
||||
pub fn insert(&mut self, ioc: IoC, verified_at: u64) {
|
||||
let stix_id = compute_stix_id(&ioc);
|
||||
let entry = VerifiedIoC {
|
||||
ioc,
|
||||
verified_at,
|
||||
stix_id,
|
||||
};
|
||||
|
||||
// Insert in sorted order (most entries append at the end)
|
||||
let pos = self
|
||||
.iocs
|
||||
.partition_point(|e| e.verified_at <= verified_at);
|
||||
self.iocs.insert(pos, entry);
|
||||
|
||||
info!(
|
||||
total = self.iocs.len(),
|
||||
verified_at,
|
||||
"IoC added to threat feed store"
|
||||
);
|
||||
}
|
||||
|
||||
/// Query IoCs with filtering and pagination.
|
||||
pub fn query(&self, params: &QueryParams) -> QueryResult {
|
||||
let filtered: Vec<&VerifiedIoC> = self
|
||||
.iocs
|
||||
.iter()
|
||||
.filter(|e| {
|
||||
if let Some(since) = params.since {
|
||||
if e.verified_at < since {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if let Some(min_sev) = params.min_severity {
|
||||
if e.ioc.severity < min_sev {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if let Some(ioc_type) = params.ioc_type {
|
||||
if e.ioc.ioc_type != ioc_type {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
true
|
||||
})
|
||||
.collect();
|
||||
|
||||
let total = filtered.len();
|
||||
let offset = params.offset.min(total);
|
||||
let limit = params.limit.min(hivemind::API_MAX_PAGE_SIZE);
|
||||
let end = (offset + limit).min(total);
|
||||
|
||||
let items: Vec<VerifiedIoC> = filtered[offset..end]
|
||||
.iter()
|
||||
.map(|e| (*e).clone())
|
||||
.collect();
|
||||
|
||||
QueryResult {
|
||||
items,
|
||||
total,
|
||||
offset,
|
||||
limit,
|
||||
}
|
||||
}
|
||||
|
||||
/// Total number of verified IoCs in the store.
|
||||
pub fn len(&self) -> usize {
|
||||
self.iocs.len()
|
||||
}
|
||||
|
||||
/// Whether the store is empty.
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.iocs.is_empty()
|
||||
}
|
||||
|
||||
/// Get all IoCs (for stats/internal use). Returns a slice reference.
|
||||
pub fn all(&self) -> &[VerifiedIoC] {
|
||||
&self.iocs
|
||||
}
|
||||
}
|
||||
|
||||
/// Parameters for querying the threat feed store.
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub struct QueryParams {
|
||||
/// Only return IoCs verified after this Unix timestamp.
|
||||
pub since: Option<u64>,
|
||||
/// Minimum severity level (0-4).
|
||||
pub min_severity: Option<u8>,
|
||||
/// Filter by IoC type.
|
||||
pub ioc_type: Option<u8>,
|
||||
/// Maximum items to return.
|
||||
pub limit: usize,
|
||||
/// Offset for pagination.
|
||||
pub offset: usize,
|
||||
}
|
||||
|
||||
impl QueryParams {
|
||||
/// Create default query with standard page size.
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
since: None,
|
||||
min_severity: None,
|
||||
ioc_type: None,
|
||||
limit: hivemind::API_DEFAULT_PAGE_SIZE,
|
||||
offset: 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Result of a store query with pagination metadata.
|
||||
#[derive(Clone, Debug, serde::Serialize)]
|
||||
pub struct QueryResult {
|
||||
/// Matching IoCs for the current page.
|
||||
pub items: Vec<VerifiedIoC>,
|
||||
/// Total matching IoCs (before pagination).
|
||||
pub total: usize,
|
||||
/// Current offset.
|
||||
pub offset: usize,
|
||||
/// Page size used.
|
||||
pub limit: usize,
|
||||
}
|
||||
|
||||
/// Compute a deterministic STIX identifier from IoC fields.
|
||||
///
|
||||
/// Format: `indicator--<uuid>` where UUID is derived from
|
||||
/// SHA256(ioc_type || ip || ja4 || first_seen).
|
||||
fn compute_stix_id(ioc: &IoC) -> String {
|
||||
let mut data = Vec::with_capacity(64);
|
||||
data.push(ioc.ioc_type);
|
||||
data.extend_from_slice(&ioc.ip.to_be_bytes());
|
||||
if let Some(ref ja4) = ioc.ja4 {
|
||||
data.extend_from_slice(ja4.as_bytes());
|
||||
}
|
||||
data.extend_from_slice(&ioc.first_seen.to_be_bytes());
|
||||
|
||||
let hash = digest::digest(&digest::SHA256, &data);
|
||||
let h = hash.as_ref();
|
||||
|
||||
// Format as UUID-like identifier (deterministic, reproducible)
|
||||
format!(
|
||||
"indicator--{:02x}{:02x}{:02x}{:02x}-{:02x}{:02x}-{:02x}{:02x}\
|
||||
-{:02x}{:02x}-{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}",
|
||||
h[0], h[1], h[2], h[3], h[4], h[5], h[6], h[7], h[8], h[9], h[10],
|
||||
h[11], h[12], h[13], h[14], h[15],
|
||||
)
|
||||
}
|
||||
|
||||
/// Convert an IPv4 u32 to dotted-decimal string.
|
||||
pub fn ip_to_string(ip: u32) -> String {
|
||||
let a = (ip >> 24) & 0xFF;
|
||||
let b = (ip >> 16) & 0xFF;
|
||||
let c = (ip >> 8) & 0xFF;
|
||||
let d = ip & 0xFF;
|
||||
format!("{a}.{b}.{c}.{d}")
|
||||
}
|
||||
|
||||
/// Convert a Unix timestamp to ISO 8601 format (YYYY-MM-DDTHH:MM:SSZ).
|
||||
///
|
||||
/// Uses Howard Hinnant's civil_from_days algorithm for calendar conversion.
|
||||
pub fn unix_to_iso8601(ts: u64) -> String {
|
||||
let time_of_day = ts % 86400;
|
||||
let hours = time_of_day / 3600;
|
||||
let minutes = (time_of_day % 3600) / 60;
|
||||
let seconds = time_of_day % 60;
|
||||
|
||||
let days = (ts / 86400) as i64;
|
||||
|
||||
// Howard Hinnant's civil_from_days algorithm
|
||||
let z = days + 719468;
|
||||
let era = if z >= 0 { z } else { z - 146096 } / 146097;
|
||||
let doe = (z - era * 146097) as u32;
|
||||
let yoe = (doe - doe / 1460 + doe / 36524 - doe / 146096) / 365;
|
||||
let y = yoe as i64 + era * 400;
|
||||
let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
|
||||
let mp = (5 * doy + 2) / 153;
|
||||
let d = doy - (153 * mp + 2) / 5 + 1;
|
||||
let m = if mp < 10 { mp + 3 } else { mp - 9 };
|
||||
let y = if m <= 2 { y + 1 } else { y };
|
||||
|
||||
format!("{y:04}-{m:02}-{d:02}T{hours:02}:{minutes:02}:{seconds:02}Z")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn make_ioc(ip: u32, severity: u8, ioc_type: u8) -> IoC {
|
||||
IoC {
|
||||
ioc_type,
|
||||
severity,
|
||||
ip,
|
||||
ja4: Some("t13d1516h2_8daaf6152771_e5627efa2ab1".to_string()),
|
||||
entropy_score: Some(7500),
|
||||
description: format!("Test IoC ip={ip}"),
|
||||
first_seen: 1700000000,
|
||||
confirmations: 3,
|
||||
zkp_proof: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn insert_and_query_all() {
|
||||
let mut store = ThreatFeedStore::new();
|
||||
store.insert(make_ioc(1, 3, 0), 1000);
|
||||
store.insert(make_ioc(2, 2, 1), 2000);
|
||||
store.insert(make_ioc(3, 4, 0), 3000);
|
||||
|
||||
let result = store.query(&QueryParams::new());
|
||||
assert_eq!(result.total, 3);
|
||||
assert_eq!(result.items.len(), 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn query_by_severity() {
|
||||
let mut store = ThreatFeedStore::new();
|
||||
store.insert(make_ioc(1, 1, 0), 1000);
|
||||
store.insert(make_ioc(2, 3, 0), 2000);
|
||||
store.insert(make_ioc(3, 4, 0), 3000);
|
||||
|
||||
let params = QueryParams {
|
||||
min_severity: Some(3),
|
||||
..QueryParams::new()
|
||||
};
|
||||
let result = store.query(¶ms);
|
||||
assert_eq!(result.total, 2);
|
||||
assert!(result.items.iter().all(|i| i.ioc.severity >= 3));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn query_by_type() {
|
||||
let mut store = ThreatFeedStore::new();
|
||||
store.insert(make_ioc(1, 3, 0), 1000);
|
||||
store.insert(make_ioc(2, 3, 1), 2000);
|
||||
store.insert(make_ioc(3, 3, 0), 3000);
|
||||
|
||||
let params = QueryParams {
|
||||
ioc_type: Some(1),
|
||||
..QueryParams::new()
|
||||
};
|
||||
let result = store.query(¶ms);
|
||||
assert_eq!(result.total, 1);
|
||||
assert_eq!(result.items[0].ioc.ioc_type, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn query_since_timestamp() {
|
||||
let mut store = ThreatFeedStore::new();
|
||||
store.insert(make_ioc(1, 3, 0), 1000);
|
||||
store.insert(make_ioc(2, 3, 0), 2000);
|
||||
store.insert(make_ioc(3, 3, 0), 3000);
|
||||
|
||||
let params = QueryParams {
|
||||
since: Some(2000),
|
||||
..QueryParams::new()
|
||||
};
|
||||
let result = store.query(¶ms);
|
||||
assert_eq!(result.total, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pagination() {
|
||||
let mut store = ThreatFeedStore::new();
|
||||
for i in 0..10 {
|
||||
store.insert(make_ioc(i, 3, 0), 1000 + u64::from(i));
|
||||
}
|
||||
|
||||
let params = QueryParams {
|
||||
limit: 3,
|
||||
offset: 2,
|
||||
..QueryParams::new()
|
||||
};
|
||||
let result = store.query(¶ms);
|
||||
assert_eq!(result.total, 10);
|
||||
assert_eq!(result.items.len(), 3);
|
||||
assert_eq!(result.offset, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stix_id_deterministic() {
|
||||
let ioc = make_ioc(0xC0A80001, 3, 0);
|
||||
let id1 = compute_stix_id(&ioc);
|
||||
let id2 = compute_stix_id(&ioc);
|
||||
assert_eq!(id1, id2);
|
||||
assert!(id1.starts_with("indicator--"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ip_conversion() {
|
||||
assert_eq!(ip_to_string(0xC0A80001), "192.168.0.1");
|
||||
assert_eq!(ip_to_string(0x0A000001), "10.0.0.1");
|
||||
assert_eq!(ip_to_string(0), "0.0.0.0");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn timestamp_conversion() {
|
||||
// 2023-11-14T22:13:20Z
|
||||
assert_eq!(unix_to_iso8601(1700000000), "2023-11-14T22:13:20Z");
|
||||
// Unix epoch
|
||||
assert_eq!(unix_to_iso8601(0), "1970-01-01T00:00:00Z");
|
||||
}
|
||||
}
|
||||
451
hivemind-api/tests/load_test.rs
Executable file
451
hivemind-api/tests/load_test.rs
Executable file
|
|
@ -0,0 +1,451 @@
|
|||
//! Load Test & Licensing Lockdown — Stress simulation for HiveMind Enterprise API.
|
||||
//!
|
||||
//! Spawns a real hyper server, seeds it with IoCs, then hammers it with
|
||||
//! concurrent clients measuring response latency and verifying tier-based
|
||||
//! access control enforcement.
|
||||
|
||||
use common::hivemind::{ApiTier, IoC};
|
||||
use hivemind_api::licensing::LicenseManager;
|
||||
use hivemind_api::server::{self, HivemindCounters};
|
||||
use hivemind_api::store::ThreatFeedStore;
|
||||
use std::net::SocketAddr;
|
||||
use std::time::Instant;
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use tokio::net::TcpStream;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Make a raw HTTP/1.1 GET request and return (status_code, body).
|
||||
async fn http_get(addr: SocketAddr, path: &str, bearer: Option<&str>) -> (u16, String) {
|
||||
let mut stream = TcpStream::connect(addr)
|
||||
.await
|
||||
.expect("TCP connect failed");
|
||||
|
||||
let mut request = format!(
|
||||
"GET {path} HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n"
|
||||
);
|
||||
if let Some(token) = bearer {
|
||||
request.push_str(&format!("Authorization: Bearer {token}\r\n"));
|
||||
}
|
||||
request.push_str("\r\n");
|
||||
|
||||
stream
|
||||
.write_all(request.as_bytes())
|
||||
.await
|
||||
.expect("write request failed");
|
||||
|
||||
let mut buf = Vec::with_capacity(8192);
|
||||
stream
|
||||
.read_to_end(&mut buf)
|
||||
.await
|
||||
.expect("read response failed");
|
||||
|
||||
let raw = String::from_utf8_lossy(&buf);
|
||||
|
||||
// Parse status code from "HTTP/1.1 NNN ..."
|
||||
let status = raw
|
||||
.get(9..12)
|
||||
.and_then(|s| s.parse::<u16>().ok())
|
||||
.unwrap_or(0);
|
||||
|
||||
// Split at the blank line separating headers from body
|
||||
let body = raw
|
||||
.find("\r\n\r\n")
|
||||
.map(|i| raw[i + 4..].to_string())
|
||||
.unwrap_or_default();
|
||||
|
||||
(status, body)
|
||||
}
|
||||
|
||||
/// Seed the store with `count` synthetic IoCs at 1-second intervals.
|
||||
fn seed_store(store: &mut ThreatFeedStore, count: usize) {
|
||||
let base_time = 1_700_000_000u64;
|
||||
for i in 0..count {
|
||||
let ioc = IoC {
|
||||
ioc_type: (i % 5) as u8,
|
||||
severity: ((i % 5) as u8).min(4),
|
||||
ip: 0xC6120000 + i as u32, // 198.18.x.x range
|
||||
ja4: if i % 3 == 0 {
|
||||
Some("t13d1516h2_8daaf6152771_e5627efa2ab1".into())
|
||||
} else {
|
||||
None
|
||||
},
|
||||
entropy_score: Some(5000 + (i as u32 * 100)),
|
||||
description: format!("Synthetic threat indicator #{i}"),
|
||||
first_seen: base_time + i as u64,
|
||||
confirmations: 3,
|
||||
zkp_proof: Vec::new(),
|
||||
};
|
||||
store.insert(ioc, base_time + i as u64 + 60);
|
||||
}
|
||||
}
|
||||
|
||||
/// Find a free TCP port by binding to :0 and reading the assigned port.
|
||||
async fn free_port() -> u16 {
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
||||
.await
|
||||
.expect("bind :0 failed");
|
||||
listener
|
||||
.local_addr()
|
||||
.expect("local_addr failed")
|
||||
.port()
|
||||
}
|
||||
|
||||
// ===========================================================================
|
||||
// Scenario A — API Hammering: 100 concurrent clients
|
||||
// ===========================================================================
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn scenario_a_api_hammering() {
|
||||
// Setup
|
||||
let store = ThreatFeedStore::shared();
|
||||
let licensing = LicenseManager::shared();
|
||||
|
||||
// Seed 50 IoCs
|
||||
{
|
||||
let mut s = store.write().expect("lock");
|
||||
seed_store(&mut s, 50);
|
||||
}
|
||||
|
||||
// Register an Enterprise-tier key
|
||||
let api_key = "test-enterprise-key-12345678";
|
||||
{
|
||||
let mut lm = licensing.write().expect("lock");
|
||||
lm.register_key(api_key, ApiTier::Enterprise);
|
||||
}
|
||||
|
||||
// Start server on a random port
|
||||
let port = free_port().await;
|
||||
let addr: SocketAddr = ([127, 0, 0, 1], port).into();
|
||||
let server_store = store.clone();
|
||||
let server_lic = licensing.clone();
|
||||
let counters = std::sync::Arc::new(HivemindCounters::default());
|
||||
let server_handle = tokio::task::spawn(async move {
|
||||
let _ = server::run(addr, server_store, server_lic, counters).await;
|
||||
});
|
||||
|
||||
// Give the server a moment to bind — try connecting in a quick retry loop
|
||||
let mut connected = false;
|
||||
for _ in 0..50 {
|
||||
if TcpStream::connect(addr).await.is_ok() {
|
||||
connected = true;
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
assert!(connected, "Server did not start within retry window");
|
||||
|
||||
// Define the endpoints to hit
|
||||
let endpoints = [
|
||||
"/api/v1/feed",
|
||||
"/api/v1/feed/stix",
|
||||
"/api/v1/feed/splunk",
|
||||
"/api/v1/stats",
|
||||
];
|
||||
|
||||
// Spawn 100 concurrent client tasks
|
||||
let client_count = 100;
|
||||
let mut handles = Vec::with_capacity(client_count);
|
||||
let start = Instant::now();
|
||||
|
||||
for i in 0..client_count {
|
||||
let endpoint = endpoints[i % endpoints.len()];
|
||||
let key = api_key.to_string();
|
||||
handles.push(tokio::task::spawn(async move {
|
||||
let t = Instant::now();
|
||||
let (status, body) = http_get(addr, endpoint, Some(&key)).await;
|
||||
let latency = t.elapsed();
|
||||
(i, status, body.len(), latency)
|
||||
}));
|
||||
}
|
||||
|
||||
// Collect results
|
||||
let mut total_latency = std::time::Duration::ZERO;
|
||||
let mut max_latency = std::time::Duration::ZERO;
|
||||
let mut error_count = 0;
|
||||
|
||||
for handle in handles {
|
||||
let (idx, status, body_len, latency) = handle.await.expect("task panicked");
|
||||
if status != 200 {
|
||||
error_count += 1;
|
||||
eprintln!(
|
||||
"[HAMMER] Client {idx}: HTTP {status} (body {body_len}B) in {latency:.2?}"
|
||||
);
|
||||
}
|
||||
total_latency += latency;
|
||||
if latency > max_latency {
|
||||
max_latency = latency;
|
||||
}
|
||||
}
|
||||
|
||||
let total_elapsed = start.elapsed();
|
||||
let avg_latency = total_latency / client_count as u32;
|
||||
|
||||
eprintln!(
|
||||
"[HAMMER] {client_count} clients completed in {total_elapsed:.2?}"
|
||||
);
|
||||
eprintln!(
|
||||
"[HAMMER] Avg latency: {avg_latency:.2?}, Max: {max_latency:.2?}, Errors: {error_count}"
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
error_count, 0,
|
||||
"All authenticated requests should succeed (HTTP 200)"
|
||||
);
|
||||
|
||||
// Abort the server
|
||||
server_handle.abort();
|
||||
}
|
||||
|
||||
// ===========================================================================
|
||||
// Scenario B — Licensing Lockdown: tier-based access denial
|
||||
// ===========================================================================
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn scenario_b_licensing_lockdown() {
|
||||
let store = ThreatFeedStore::shared();
|
||||
let licensing = LicenseManager::shared();
|
||||
|
||||
// Seed 10 IoCs
|
||||
{
|
||||
let mut s = store.write().expect("lock");
|
||||
seed_store(&mut s, 10);
|
||||
}
|
||||
|
||||
// Register keys at each tier
|
||||
let free_key = "free-tier-key-aaaa";
|
||||
let enterprise_key = "enterprise-tier-key-bbbb";
|
||||
let ns_key = "national-security-key-cccc";
|
||||
{
|
||||
let mut lm = licensing.write().expect("lock");
|
||||
lm.register_key(free_key, ApiTier::Free);
|
||||
lm.register_key(enterprise_key, ApiTier::Enterprise);
|
||||
lm.register_key(ns_key, ApiTier::NationalSecurity);
|
||||
}
|
||||
|
||||
let port = free_port().await;
|
||||
let addr: SocketAddr = ([127, 0, 0, 1], port).into();
|
||||
let server_store = store.clone();
|
||||
let server_lic = licensing.clone();
|
||||
let counters = std::sync::Arc::new(HivemindCounters::default());
|
||||
let server_handle = tokio::task::spawn(async move {
|
||||
let _ = server::run(addr, server_store, server_lic, counters).await;
|
||||
});
|
||||
|
||||
// Wait for server
|
||||
for _ in 0..50 {
|
||||
if TcpStream::connect(addr).await.is_ok() {
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
|
||||
// --- Free tier: /api/v1/feed should work ---
|
||||
let (status, _) = http_get(addr, "/api/v1/feed", Some(free_key)).await;
|
||||
assert_eq!(status, 200, "Free tier should access /api/v1/feed");
|
||||
|
||||
// --- Free tier: /api/v1/stats should work ---
|
||||
let (status, _) = http_get(addr, "/api/v1/stats", Some(free_key)).await;
|
||||
assert_eq!(status, 200, "Free tier should access /api/v1/stats");
|
||||
|
||||
// --- Free tier: SIEM endpoints BLOCKED ---
|
||||
let siem_paths = [
|
||||
"/api/v1/feed/splunk",
|
||||
"/api/v1/feed/qradar",
|
||||
"/api/v1/feed/cef",
|
||||
];
|
||||
for path in &siem_paths {
|
||||
let (status, _) = http_get(addr, path, Some(free_key)).await;
|
||||
assert_eq!(
|
||||
status, 403,
|
||||
"Free tier should be FORBIDDEN from {path}"
|
||||
);
|
||||
}
|
||||
|
||||
// --- Free tier: STIX/TAXII endpoints BLOCKED ---
|
||||
let taxii_paths = [
|
||||
"/api/v1/feed/stix",
|
||||
"/taxii2/collections/",
|
||||
];
|
||||
for path in &taxii_paths {
|
||||
let (status, _) = http_get(addr, path, Some(free_key)).await;
|
||||
assert_eq!(
|
||||
status, 403,
|
||||
"Free tier should be FORBIDDEN from {path}"
|
||||
);
|
||||
}
|
||||
|
||||
// --- No auth: should get 401 Unauthorized ---
|
||||
let (status, _) = http_get(addr, "/api/v1/feed/splunk", None).await;
|
||||
assert_eq!(
|
||||
status, 401,
|
||||
"No auth header should yield 401 Unauthorized"
|
||||
);
|
||||
|
||||
// --- Invalid key: should get denied ---
|
||||
let (status, _) = http_get(addr, "/api/v1/feed", Some("totally-bogus-key")).await;
|
||||
// /api/v1/feed allows unauthenticated access via effective_tier=Free fallback,
|
||||
// but with an invalid key, the server resolves tier to None and falls through
|
||||
// to Free default for /api/v1/feed
|
||||
assert!(
|
||||
status == 200 || status == 401,
|
||||
"/api/v1/feed with invalid key: got {status}"
|
||||
);
|
||||
|
||||
// --- Enterprise tier: SIEM endpoints ALLOWED ---
|
||||
for path in &siem_paths {
|
||||
let (status, body) = http_get(addr, path, Some(enterprise_key)).await;
|
||||
assert_eq!(
|
||||
status, 200,
|
||||
"Enterprise tier should access {path}, got {status}"
|
||||
);
|
||||
assert!(!body.is_empty(), "{path} response body should not be empty");
|
||||
}
|
||||
|
||||
// --- Enterprise tier: TAXII endpoints ALLOWED ---
|
||||
for path in &taxii_paths {
|
||||
let (status, _) = http_get(addr, path, Some(enterprise_key)).await;
|
||||
assert_eq!(
|
||||
status, 200,
|
||||
"Enterprise tier should access {path}, got {status}"
|
||||
);
|
||||
}
|
||||
|
||||
// --- NationalSecurity tier: everything ALLOWED ---
|
||||
let all_paths = [
|
||||
"/api/v1/feed",
|
||||
"/api/v1/feed/stix",
|
||||
"/api/v1/feed/splunk",
|
||||
"/api/v1/feed/qradar",
|
||||
"/api/v1/feed/cef",
|
||||
"/api/v1/stats",
|
||||
"/taxii2/",
|
||||
"/taxii2/collections/",
|
||||
];
|
||||
for path in &all_paths {
|
||||
let (status, _) = http_get(addr, path, Some(ns_key)).await;
|
||||
assert_eq!(
|
||||
status, 200,
|
||||
"NationalSecurity tier should access {path}, got {status}"
|
||||
);
|
||||
}
|
||||
|
||||
// --- Unknown endpoint: 404 ---
|
||||
let (status, _) = http_get(addr, "/api/v1/nonexistent", Some(enterprise_key)).await;
|
||||
assert_eq!(status, 404, "Unknown endpoint should yield 404");
|
||||
|
||||
eprintln!("[LOCKDOWN] All tier-based access control assertions passed");
|
||||
server_handle.abort();
|
||||
}
|
||||
|
||||
// ===========================================================================
|
||||
// Scenario C — Feed Content Integrity: verify response payloads
|
||||
// ===========================================================================
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn scenario_c_feed_content_integrity() {
|
||||
let store = ThreatFeedStore::shared();
|
||||
let licensing = LicenseManager::shared();
|
||||
|
||||
// Seed 5 IoCs
|
||||
{
|
||||
let mut s = store.write().expect("lock");
|
||||
seed_store(&mut s, 5);
|
||||
}
|
||||
|
||||
let api_key = "integrity-test-key";
|
||||
{
|
||||
let mut lm = licensing.write().expect("lock");
|
||||
lm.register_key(api_key, ApiTier::Enterprise);
|
||||
}
|
||||
|
||||
let port = free_port().await;
|
||||
let addr: SocketAddr = ([127, 0, 0, 1], port).into();
|
||||
let ss = store.clone();
|
||||
let sl = licensing.clone();
|
||||
let counters = std::sync::Arc::new(HivemindCounters::default());
|
||||
let server_handle = tokio::task::spawn(async move {
|
||||
let _ = server::run(addr, ss, sl, counters).await;
|
||||
});
|
||||
|
||||
for _ in 0..50 {
|
||||
if TcpStream::connect(addr).await.is_ok() {
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
|
||||
// --- JSON feed ---
|
||||
let (status, body) = http_get(addr, "/api/v1/feed", Some(api_key)).await;
|
||||
assert_eq!(status, 200);
|
||||
let parsed: serde_json::Value = serde_json::from_str(&body)
|
||||
.unwrap_or_else(|e| panic!("Invalid JSON in /api/v1/feed response: {e}"));
|
||||
assert!(
|
||||
parsed.get("items").is_some(),
|
||||
"Feed response should contain 'items' field"
|
||||
);
|
||||
assert!(
|
||||
parsed.get("total").is_some(),
|
||||
"Feed response should contain 'total' field"
|
||||
);
|
||||
|
||||
// --- STIX bundle ---
|
||||
let (status, body) = http_get(addr, "/api/v1/feed/stix", Some(api_key)).await;
|
||||
assert_eq!(status, 200);
|
||||
let stix: serde_json::Value = serde_json::from_str(&body)
|
||||
.unwrap_or_else(|e| panic!("Invalid STIX JSON: {e}"));
|
||||
assert_eq!(
|
||||
stix.get("type").and_then(|t| t.as_str()),
|
||||
Some("bundle"),
|
||||
"STIX response should be a bundle"
|
||||
);
|
||||
|
||||
// --- Splunk HEC ---
|
||||
let (status, body) = http_get(addr, "/api/v1/feed/splunk", Some(api_key)).await;
|
||||
assert_eq!(status, 200);
|
||||
let splunk: serde_json::Value = serde_json::from_str(&body)
|
||||
.unwrap_or_else(|e| panic!("Invalid Splunk JSON: {e}"));
|
||||
assert!(splunk.is_array(), "Splunk response should be a JSON array");
|
||||
|
||||
// --- QRadar LEEF (plain text) ---
|
||||
let (status, body) = http_get(addr, "/api/v1/feed/qradar", Some(api_key)).await;
|
||||
assert_eq!(status, 200);
|
||||
assert!(
|
||||
body.contains("LEEF:"),
|
||||
"QRadar response should contain LEEF headers"
|
||||
);
|
||||
|
||||
// --- CEF (plain text) ---
|
||||
let (status, body) = http_get(addr, "/api/v1/feed/cef", Some(api_key)).await;
|
||||
assert_eq!(status, 200);
|
||||
assert!(
|
||||
body.contains("CEF:"),
|
||||
"CEF response should contain CEF headers"
|
||||
);
|
||||
|
||||
// --- Stats ---
|
||||
let (status, body) = http_get(addr, "/api/v1/stats", Some(api_key)).await;
|
||||
assert_eq!(status, 200);
|
||||
let stats: serde_json::Value = serde_json::from_str(&body)
|
||||
.unwrap_or_else(|e| panic!("Invalid stats JSON: {e}"));
|
||||
let total = stats
|
||||
.get("total_iocs")
|
||||
.and_then(|v| v.as_u64())
|
||||
.unwrap_or(0);
|
||||
assert_eq!(total, 5, "Stats should report 5 total IoCs");
|
||||
|
||||
// --- TAXII discovery (no auth needed, but we send one) ---
|
||||
let (status, body) = http_get(addr, "/taxii2/", Some(api_key)).await;
|
||||
assert_eq!(status, 200);
|
||||
let taxii: serde_json::Value = serde_json::from_str(&body)
|
||||
.unwrap_or_else(|e| panic!("Invalid TAXII JSON: {e}"));
|
||||
assert!(
|
||||
taxii.get("title").is_some(),
|
||||
"TAXII discovery should contain title"
|
||||
);
|
||||
|
||||
eprintln!("[INTEGRITY] All 7 endpoints return well-formed responses");
|
||||
server_handle.abort();
|
||||
}
|
||||
17
hivemind-dashboard/Cargo.toml
Executable file
17
hivemind-dashboard/Cargo.toml
Executable file
|
|
@ -0,0 +1,17 @@
|
|||
[package]
|
||||
name = "hivemind-dashboard"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
description = "Live TUI dashboard for HiveMind mesh monitoring"
|
||||
|
||||
[dependencies]
|
||||
common = { path = "../common", default-features = false, features = ["user"] }
|
||||
tokio = { workspace = true }
|
||||
hyper = { workspace = true }
|
||||
http-body-util = { workspace = true }
|
||||
hyper-util = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
anyhow = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
tracing-subscriber = { workspace = true }
|
||||
345
hivemind-dashboard/src/app.rs
Executable file
345
hivemind-dashboard/src/app.rs
Executable file
|
|
@ -0,0 +1,345 @@
|
|||
//! ANSI terminal dashboard renderer.
|
||||
//!
|
||||
//! Renders a live-updating dashboard using only ANSI escape codes.
|
||||
//! No external TUI crates — pure `\x1B[` sequences.
|
||||
|
||||
use std::io::{self, Write};
|
||||
|
||||
use crate::collector::MeshStats;
|
||||
|
||||
/// ANSI color codes.
|
||||
const GREEN: &str = "\x1B[32m";
|
||||
const YELLOW: &str = "\x1B[33m";
|
||||
const RED: &str = "\x1B[31m";
|
||||
const CYAN: &str = "\x1B[36m";
|
||||
const BOLD: &str = "\x1B[1m";
|
||||
const DIM: &str = "\x1B[2m";
|
||||
const RESET: &str = "\x1B[0m";
|
||||
|
||||
/// Unicode box-drawing characters.
|
||||
const TL: char = '┌';
|
||||
const TR: char = '┐';
|
||||
const BL: char = '└';
|
||||
const BR: char = '┘';
|
||||
const HZ: char = '─';
|
||||
const VT: char = '│';
|
||||
|
||||
/// Terminal dashboard state.
|
||||
pub struct Dashboard {
|
||||
stats: MeshStats,
|
||||
frame: u64,
|
||||
}
|
||||
|
||||
impl Dashboard {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
stats: MeshStats::default(),
|
||||
frame: 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// Update dashboard with fresh stats.
|
||||
pub fn update(&mut self, stats: MeshStats) {
|
||||
self.stats = stats;
|
||||
self.frame += 1;
|
||||
}
|
||||
|
||||
/// Render the full dashboard to the given writer.
|
||||
pub fn render<W: Write>(&self, w: &mut W) -> io::Result<()> {
|
||||
// Move cursor to top-left, clear screen
|
||||
write!(w, "\x1B[H\x1B[2J")?;
|
||||
|
||||
self.render_header(w)?;
|
||||
writeln!(w)?;
|
||||
self.render_mesh_panel(w)?;
|
||||
writeln!(w)?;
|
||||
self.render_threat_panel(w)?;
|
||||
writeln!(w)?;
|
||||
self.render_network_fw_panel(w)?;
|
||||
writeln!(w)?;
|
||||
self.render_a2a_panel(w)?;
|
||||
writeln!(w)?;
|
||||
self.render_crypto_panel(w)?;
|
||||
writeln!(w)?;
|
||||
self.render_footer(w)?;
|
||||
|
||||
w.flush()
|
||||
}
|
||||
|
||||
fn render_header<W: Write>(&self, w: &mut W) -> io::Result<()> {
|
||||
let status = if self.stats.connected {
|
||||
format!("{GREEN}● CONNECTED{RESET}")
|
||||
} else {
|
||||
format!("{RED}○ DISCONNECTED{RESET}")
|
||||
};
|
||||
|
||||
writeln!(
|
||||
w,
|
||||
" {BOLD}{CYAN}╔══════════════════════════════════════════════════╗{RESET}"
|
||||
)?;
|
||||
writeln!(
|
||||
w,
|
||||
" {BOLD}{CYAN}║{RESET} {BOLD}BLACKWALL HIVEMIND{RESET} {DIM}v2.0{RESET} \
|
||||
{status} {DIM}frame #{}{RESET} {BOLD}{CYAN}║{RESET}",
|
||||
self.frame,
|
||||
)?;
|
||||
writeln!(
|
||||
w,
|
||||
" {BOLD}{CYAN}╚══════════════════════════════════════════════════╝{RESET}"
|
||||
)
|
||||
}
|
||||
|
||||
fn render_mesh_panel<W: Write>(&self, w: &mut W) -> io::Result<()> {
|
||||
self.draw_box_top(w, " P2P Mesh ", 48)?;
|
||||
self.draw_kv(w, "Peers", &self.stats.peer_count.to_string(), 48)?;
|
||||
self.draw_kv(
|
||||
w,
|
||||
"DHT Records",
|
||||
&self.stats.dht_records.to_string(),
|
||||
48,
|
||||
)?;
|
||||
self.draw_kv(
|
||||
w,
|
||||
"GossipSub Topics",
|
||||
&self.stats.gossip_topics.to_string(),
|
||||
48,
|
||||
)?;
|
||||
self.draw_kv(
|
||||
w,
|
||||
"Messages/s",
|
||||
&format!("{:.1}", self.stats.messages_per_sec),
|
||||
48,
|
||||
)?;
|
||||
self.draw_box_bottom(w, 48)
|
||||
}
|
||||
|
||||
fn render_threat_panel<W: Write>(&self, w: &mut W) -> io::Result<()> {
|
||||
self.draw_box_top(w, " Threat Intel ", 48)?;
|
||||
self.draw_kv(
|
||||
w,
|
||||
"IoCs Shared",
|
||||
&self.stats.iocs_shared.to_string(),
|
||||
48,
|
||||
)?;
|
||||
self.draw_kv(
|
||||
w,
|
||||
"IoCs Received",
|
||||
&self.stats.iocs_received.to_string(),
|
||||
48,
|
||||
)?;
|
||||
|
||||
let reputation_color = if self.stats.avg_reputation > 80.0 {
|
||||
GREEN
|
||||
} else if self.stats.avg_reputation > 50.0 {
|
||||
YELLOW
|
||||
} else {
|
||||
RED
|
||||
};
|
||||
self.draw_kv_colored(
|
||||
w,
|
||||
"Avg Reputation",
|
||||
&format!("{:.1}", self.stats.avg_reputation),
|
||||
reputation_color,
|
||||
48,
|
||||
)?;
|
||||
self.draw_box_bottom(w, 48)
|
||||
}
|
||||
|
||||
fn render_a2a_panel<W: Write>(&self, w: &mut W) -> io::Result<()> {
|
||||
self.draw_box_top(w, " A2A Firewall ", 48)?;
|
||||
self.draw_kv(
|
||||
w,
|
||||
"JWTs Verified",
|
||||
&self.stats.a2a_jwts_verified.to_string(),
|
||||
48,
|
||||
)?;
|
||||
self.draw_kv(
|
||||
w,
|
||||
"Violations Blocked",
|
||||
&self.stats.a2a_violations.to_string(),
|
||||
48,
|
||||
)?;
|
||||
self.draw_kv(
|
||||
w,
|
||||
"Injections Detected",
|
||||
&self.stats.a2a_injections.to_string(),
|
||||
48,
|
||||
)?;
|
||||
self.draw_box_bottom(w, 48)
|
||||
}
|
||||
|
||||
fn render_network_fw_panel<W: Write>(&self, w: &mut W) -> io::Result<()> {
|
||||
self.draw_box_top(w, " Network Firewall ", 48)?;
|
||||
self.draw_kv(
|
||||
w,
|
||||
"Packets Total",
|
||||
&self.stats.packets_total.to_string(),
|
||||
48,
|
||||
)?;
|
||||
self.draw_kv(
|
||||
w,
|
||||
"Packets Passed",
|
||||
&self.stats.packets_passed.to_string(),
|
||||
48,
|
||||
)?;
|
||||
|
||||
let dropped_color = if self.stats.packets_dropped > 0 {
|
||||
RED
|
||||
} else {
|
||||
GREEN
|
||||
};
|
||||
self.draw_kv_colored(
|
||||
w,
|
||||
"Packets Dropped",
|
||||
&self.stats.packets_dropped.to_string(),
|
||||
dropped_color,
|
||||
48,
|
||||
)?;
|
||||
|
||||
let anomaly_color = if self.stats.anomalies_sent > 0 {
|
||||
YELLOW
|
||||
} else {
|
||||
GREEN
|
||||
};
|
||||
self.draw_kv_colored(
|
||||
w,
|
||||
"Anomalies",
|
||||
&self.stats.anomalies_sent.to_string(),
|
||||
anomaly_color,
|
||||
48,
|
||||
)?;
|
||||
self.draw_box_bottom(w, 48)
|
||||
}
|
||||
|
||||
fn render_crypto_panel<W: Write>(&self, w: &mut W) -> io::Result<()> {
|
||||
self.draw_box_top(w, " Cryptography ", 48)?;
|
||||
self.draw_kv(
|
||||
w,
|
||||
"ZKP Proofs Generated",
|
||||
&self.stats.zkp_proofs_generated.to_string(),
|
||||
48,
|
||||
)?;
|
||||
self.draw_kv(
|
||||
w,
|
||||
"ZKP Proofs Verified",
|
||||
&self.stats.zkp_proofs_verified.to_string(),
|
||||
48,
|
||||
)?;
|
||||
|
||||
let fhe_status = if self.stats.fhe_encrypted {
|
||||
format!("{GREEN}AES-256-GCM{RESET}")
|
||||
} else {
|
||||
format!("{YELLOW}STUB{RESET}")
|
||||
};
|
||||
self.draw_kv(w, "FHE Mode", &fhe_status, 48)?;
|
||||
self.draw_box_bottom(w, 48)
|
||||
}
|
||||
|
||||
fn render_footer<W: Write>(&self, w: &mut W) -> io::Result<()> {
|
||||
writeln!(
|
||||
w,
|
||||
" {DIM}Press Ctrl+C to exit | Refresh: 1s | \
|
||||
API: hivemind-api{RESET}"
|
||||
)
|
||||
}
|
||||
|
||||
// ── Box drawing helpers ─────────────────────────────────────
|
||||
|
||||
fn draw_box_top<W: Write>(
|
||||
&self,
|
||||
w: &mut W,
|
||||
title: &str,
|
||||
width: usize,
|
||||
) -> io::Result<()> {
|
||||
let inner = width - 2 - title.len();
|
||||
write!(w, " {TL}{HZ}")?;
|
||||
write!(w, "{BOLD}{title}{RESET}")?;
|
||||
for _ in 0..inner {
|
||||
write!(w, "{HZ}")?;
|
||||
}
|
||||
writeln!(w, "{TR}")
|
||||
}
|
||||
|
||||
fn draw_box_bottom<W: Write>(&self, w: &mut W, width: usize) -> io::Result<()> {
|
||||
write!(w, " {BL}")?;
|
||||
for _ in 0..width - 2 {
|
||||
write!(w, "{HZ}")?;
|
||||
}
|
||||
writeln!(w, "{BR}")
|
||||
}
|
||||
|
||||
fn draw_kv<W: Write>(
|
||||
&self,
|
||||
w: &mut W,
|
||||
key: &str,
|
||||
value: &str,
|
||||
width: usize,
|
||||
) -> io::Result<()> {
|
||||
let padding = width - 6 - key.len() - value.len();
|
||||
let pad = if padding > 0 { padding } else { 1 };
|
||||
write!(w, " {VT} {key}")?;
|
||||
for _ in 0..pad {
|
||||
write!(w, " ")?;
|
||||
}
|
||||
writeln!(w, "{BOLD}{value}{RESET} {VT}")
|
||||
}
|
||||
|
||||
fn draw_kv_colored<W: Write>(
|
||||
&self,
|
||||
w: &mut W,
|
||||
key: &str,
|
||||
value: &str,
|
||||
color: &str,
|
||||
width: usize,
|
||||
) -> io::Result<()> {
|
||||
let padding = width - 6 - key.len() - value.len();
|
||||
let pad = if padding > 0 { padding } else { 1 };
|
||||
write!(w, " {VT} {key}")?;
|
||||
for _ in 0..pad {
|
||||
write!(w, " ")?;
|
||||
}
|
||||
writeln!(w, "{color}{BOLD}{value}{RESET} {VT}")
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn render_default_dashboard() {
|
||||
let dash = Dashboard::new();
|
||||
let mut buf = Vec::new();
|
||||
dash.render(&mut buf).expect("render");
|
||||
let output = String::from_utf8(buf).expect("utf8");
|
||||
assert!(output.contains("BLACKWALL HIVEMIND"));
|
||||
assert!(output.contains("P2P Mesh"));
|
||||
assert!(output.contains("Threat Intel"));
|
||||
assert!(output.contains("Network Firewall"));
|
||||
assert!(output.contains("A2A Firewall"));
|
||||
assert!(output.contains("Cryptography"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn update_increments_frame() {
|
||||
let mut dash = Dashboard::new();
|
||||
assert_eq!(dash.frame, 0);
|
||||
dash.update(MeshStats::default());
|
||||
assert_eq!(dash.frame, 1);
|
||||
dash.update(MeshStats::default());
|
||||
assert_eq!(dash.frame, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn connected_shows_green() {
|
||||
let mut dash = Dashboard::new();
|
||||
dash.update(MeshStats {
|
||||
connected: true,
|
||||
..Default::default()
|
||||
});
|
||||
let mut buf = Vec::new();
|
||||
dash.render(&mut buf).expect("render");
|
||||
let output = String::from_utf8(buf).expect("utf8");
|
||||
assert!(output.contains("CONNECTED"));
|
||||
}
|
||||
}
|
||||
150
hivemind-dashboard/src/collector.rs
Executable file
150
hivemind-dashboard/src/collector.rs
Executable file
|
|
@ -0,0 +1,150 @@
|
|||
//! Stats collector — polls hivemind-api for mesh statistics.
|
||||
|
||||
use serde::Deserialize;
|
||||
use tracing::warn;
|
||||
|
||||
/// Collected mesh statistics for dashboard rendering.
|
||||
#[derive(Debug, Clone, Default, Deserialize)]
|
||||
pub struct MeshStats {
|
||||
/// Whether we're connected to the API.
|
||||
#[serde(default)]
|
||||
pub connected: bool,
|
||||
|
||||
// P2P Mesh
|
||||
#[serde(default)]
|
||||
pub peer_count: u64,
|
||||
#[serde(default)]
|
||||
pub dht_records: u64,
|
||||
#[serde(default)]
|
||||
pub gossip_topics: u64,
|
||||
#[serde(default)]
|
||||
pub messages_per_sec: f64,
|
||||
|
||||
// Threat Intel
|
||||
#[serde(default)]
|
||||
pub iocs_shared: u64,
|
||||
#[serde(default)]
|
||||
pub iocs_received: u64,
|
||||
#[serde(default)]
|
||||
pub avg_reputation: f64,
|
||||
|
||||
// Network Firewall (XDP/eBPF)
|
||||
#[serde(default)]
|
||||
pub packets_total: u64,
|
||||
#[serde(default)]
|
||||
pub packets_passed: u64,
|
||||
#[serde(default)]
|
||||
pub packets_dropped: u64,
|
||||
#[serde(default)]
|
||||
pub anomalies_sent: u64,
|
||||
|
||||
// A2A Firewall (separate counters)
|
||||
#[serde(default)]
|
||||
pub a2a_jwts_verified: u64,
|
||||
#[serde(default)]
|
||||
pub a2a_violations: u64,
|
||||
#[serde(default)]
|
||||
pub a2a_injections: u64,
|
||||
|
||||
// Cryptography
|
||||
#[serde(default)]
|
||||
pub zkp_proofs_generated: u64,
|
||||
#[serde(default)]
|
||||
pub zkp_proofs_verified: u64,
|
||||
#[serde(default)]
|
||||
pub fhe_encrypted: bool,
|
||||
}
|
||||
|
||||
/// HTTP collector that polls the hivemind-api stats endpoint.
|
||||
pub struct Collector {
|
||||
url: String,
|
||||
}
|
||||
|
||||
impl Collector {
|
||||
/// Create a new collector targeting the given base URL.
|
||||
pub fn new(base_url: &str) -> Self {
|
||||
Self {
|
||||
url: format!("{}/stats", base_url.trim_end_matches('/')),
|
||||
}
|
||||
}
|
||||
|
||||
/// Fetch latest stats from the API.
|
||||
///
|
||||
/// Returns default stats with `connected = false` on any error.
|
||||
pub async fn fetch(&self) -> MeshStats {
|
||||
match self.fetch_inner().await {
|
||||
Ok(mut stats) => {
|
||||
stats.connected = true;
|
||||
stats
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(url = %self.url, error = %e, "failed to fetch mesh stats");
|
||||
MeshStats {
|
||||
connected: false,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn fetch_inner(&self) -> Result<MeshStats, Box<dyn std::error::Error>> {
|
||||
// Use a simple TCP connection + manual HTTP/1.1 request
|
||||
// to avoid pulling in heavy HTTP client deps.
|
||||
let url: hyper::Uri = self.url.parse()?;
|
||||
let host = url.host().unwrap_or("127.0.0.1");
|
||||
let port = url.port_u16().unwrap_or(9100);
|
||||
let path = url.path();
|
||||
|
||||
let stream = tokio::net::TcpStream::connect(format!("{host}:{port}")).await?;
|
||||
let io = hyper_util::rt::TokioIo::new(stream);
|
||||
|
||||
let (mut sender, conn) = hyper::client::conn::http1::handshake(io).await?;
|
||||
tokio::spawn(async move {
|
||||
if let Err(e) = conn.await {
|
||||
warn!(error = %e, "HTTP connection error");
|
||||
}
|
||||
});
|
||||
|
||||
let req = hyper::Request::builder()
|
||||
.uri(path)
|
||||
.header("Host", host)
|
||||
.body(http_body_util::Empty::<hyper::body::Bytes>::new())?;
|
||||
|
||||
let resp = sender.send_request(req).await?;
|
||||
|
||||
use http_body_util::BodyExt;
|
||||
let body = resp.into_body().collect().await?.to_bytes();
|
||||
let stats: MeshStats = serde_json::from_slice(&body)?;
|
||||
Ok(stats)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn default_stats_disconnected() {
|
||||
let stats = MeshStats::default();
|
||||
assert!(!stats.connected);
|
||||
assert_eq!(stats.peer_count, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deserialize_partial_json() {
|
||||
let json = r#"{"peer_count": 42, "fhe_encrypted": true}"#;
|
||||
let stats: MeshStats = serde_json::from_str(json).expect("parse");
|
||||
assert_eq!(stats.peer_count, 42);
|
||||
assert!(stats.fhe_encrypted);
|
||||
assert_eq!(stats.a2a_jwts_verified, 0); // default
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn collector_url_construction() {
|
||||
let c1 = Collector::new("http://localhost:9100");
|
||||
assert_eq!(c1.url, "http://localhost:9100/stats");
|
||||
|
||||
let c2 = Collector::new("http://localhost:9100/");
|
||||
assert_eq!(c2.url, "http://localhost:9100/stats");
|
||||
}
|
||||
}
|
||||
76
hivemind-dashboard/src/main.rs
Executable file
76
hivemind-dashboard/src/main.rs
Executable file
|
|
@ -0,0 +1,76 @@
|
|||
//! HiveMind TUI Dashboard — live mesh monitoring via ANSI terminal.
|
||||
//!
|
||||
//! Zero external TUI deps — pure ANSI escape codes + tokio.
|
||||
//! Polls the hivemind-api HTTP endpoint for stats and renders
|
||||
//! a live-updating dashboard in the terminal.
|
||||
|
||||
use std::io::{self, Write};
|
||||
use std::time::Duration;
|
||||
use tokio::signal;
|
||||
use tracing_subscriber::EnvFilter;
|
||||
|
||||
mod app;
|
||||
mod collector;
|
||||
|
||||
use app::Dashboard;
|
||||
use collector::Collector;
|
||||
|
||||
/// Default refresh interval in milliseconds.
|
||||
const DEFAULT_REFRESH_MS: u64 = 1000;
|
||||
|
||||
/// Default API endpoint (hivemind-api default port).
|
||||
const DEFAULT_API_URL: &str = "http://127.0.0.1:8090";
|
||||
|
||||
#[tokio::main(flavor = "current_thread")]
|
||||
async fn main() -> anyhow::Result<()> {
|
||||
tracing_subscriber::fmt()
|
||||
.with_env_filter(EnvFilter::from_default_env())
|
||||
.with_target(false)
|
||||
.init();
|
||||
|
||||
let api_url = std::env::args()
|
||||
.nth(1)
|
||||
.unwrap_or_else(|| DEFAULT_API_URL.to_string());
|
||||
|
||||
let refresh = Duration::from_millis(
|
||||
std::env::args()
|
||||
.nth(2)
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(DEFAULT_REFRESH_MS),
|
||||
);
|
||||
|
||||
let collector = Collector::new(&api_url);
|
||||
let mut dashboard = Dashboard::new();
|
||||
|
||||
// Enter alternate screen + hide cursor
|
||||
print!("\x1B[?1049h\x1B[?25l");
|
||||
io::stdout().flush()?;
|
||||
|
||||
let result = run_loop(&collector, &mut dashboard, refresh).await;
|
||||
|
||||
// Restore terminal: show cursor + leave alternate screen
|
||||
print!("\x1B[?25h\x1B[?1049l");
|
||||
io::stdout().flush()?;
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
async fn run_loop(
|
||||
collector: &Collector,
|
||||
dashboard: &mut Dashboard,
|
||||
refresh: Duration,
|
||||
) -> anyhow::Result<()> {
|
||||
loop {
|
||||
tokio::select! {
|
||||
_ = signal::ctrl_c() => {
|
||||
break;
|
||||
}
|
||||
_ = tokio::time::sleep(refresh) => {
|
||||
let stats = collector.fetch().await;
|
||||
dashboard.update(stats);
|
||||
dashboard.render(&mut io::stdout())?;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
50
hivemind/Cargo.toml
Executable file
50
hivemind/Cargo.toml
Executable file
|
|
@ -0,0 +1,50 @@
|
|||
[package]
|
||||
name = "hivemind"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
description = "HiveMind Threat Mesh — decentralized P2P threat intelligence network"
|
||||
|
||||
[features]
|
||||
default = []
|
||||
# Feature-gated real Groth16 ZK-SNARK circuits (requires bellman + bls12_381).
|
||||
# V1.0 uses ring-only commit-and-sign. Enable for Phase 2+.
|
||||
zkp-groth16 = ["bellman", "bls12_381"]
|
||||
# Feature-gated real FHE for encrypted gradient aggregation (requires tfhe).
|
||||
# V1.0 uses AES-256-GCM (not homomorphic). Enable for Phase 3+.
|
||||
fhe-real = ["tfhe"]
|
||||
|
||||
[dependencies]
|
||||
common = { path = "../common", default-features = false, features = ["user"] }
|
||||
tokio = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
tracing-subscriber = { workspace = true }
|
||||
anyhow = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
toml = { workspace = true }
|
||||
ring = { workspace = true }
|
||||
hyper = { workspace = true }
|
||||
hyper-util = { workspace = true }
|
||||
http-body-util = { workspace = true }
|
||||
nix = { workspace = true, features = ["user"] }
|
||||
libp2p = { version = "0.54", features = [
|
||||
"tokio",
|
||||
"quic",
|
||||
"noise",
|
||||
"gossipsub",
|
||||
"kad",
|
||||
"mdns",
|
||||
"macros",
|
||||
"identify",
|
||||
] }
|
||||
|
||||
# ZKP dependencies (feature-gated)
|
||||
bellman = { version = "0.14", optional = true }
|
||||
bls12_381 = { version = "0.8", optional = true }
|
||||
|
||||
# FHE dependency (feature-gated) — real homomorphic encryption
|
||||
tfhe = { version = "0.8", optional = true, features = ["shortint", "x86_64-unix"] }
|
||||
|
||||
[[bin]]
|
||||
name = "hivemind"
|
||||
path = "src/main.rs"
|
||||
232
hivemind/src/bootstrap.rs
Executable file
232
hivemind/src/bootstrap.rs
Executable file
|
|
@ -0,0 +1,232 @@
|
|||
/// Bootstrap protocol for HiveMind peer discovery.
|
||||
///
|
||||
/// Supports three discovery mechanisms:
|
||||
/// 1. Hardcoded bootstrap nodes (compiled into the binary)
|
||||
/// 2. User-configured bootstrap nodes (from hivemind.toml)
|
||||
/// 3. mDNS (for local/LAN peer discovery)
|
||||
///
|
||||
/// ARCH: Bootstrap nodes will become Circuit Relay v2 servers
|
||||
/// for NAT traversal once AutoNAT is integrated.
|
||||
use anyhow::Context;
|
||||
use libp2p::{Multiaddr, PeerId, Swarm};
|
||||
use libp2p::multiaddr::Protocol;
|
||||
use tracing::{info, warn};
|
||||
|
||||
use crate::config::HiveMindConfig;
|
||||
use crate::transport::HiveMindBehaviour;
|
||||
|
||||
/// Check if a multiaddress points to a routable (non-loopback, non-unspecified) IP.
|
||||
///
|
||||
/// Rejects 127.0.0.0/8, ::1, 0.0.0.0, :: to prevent self-referencing connections
|
||||
/// that cause "Unexpected peer ID" errors in Kademlia.
|
||||
pub fn is_routable_addr(addr: &Multiaddr) -> bool {
|
||||
let mut has_ip = false;
|
||||
for proto in addr.iter() {
|
||||
match proto {
|
||||
Protocol::Ip4(ip) => {
|
||||
has_ip = true;
|
||||
if ip.is_loopback() || ip.is_unspecified() {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
Protocol::Ip6(ip) => {
|
||||
has_ip = true;
|
||||
if ip.is_loopback() || ip.is_unspecified() {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
has_ip
|
||||
}
|
||||
|
||||
/// Built-in bootstrap nodes baked into the binary.
|
||||
///
|
||||
/// These are lightweight relay-only VPS instances that never go down.
|
||||
/// They run `mode = "bootstrap"` and serve only as DHT entry points.
|
||||
/// Users can disable them by setting `bootstrap.use_default_nodes = false`.
|
||||
///
|
||||
/// IMPORTANT: Update these when deploying new bootstrap infrastructure.
|
||||
/// Format: "/dns4/<hostname>/udp/4001/quic-v1/p2p/<peer-id>"
|
||||
///
|
||||
/// Placeholder entries below — replace with real VPS PeerIds after
|
||||
/// first deployment. The nodes won't connect until real PeerIds exist,
|
||||
/// which is safe (they just log a warning and fall back to mDNS).
|
||||
pub const DEFAULT_BOOTSTRAP_NODES: &[&str] = &[
|
||||
// EU-West (Amsterdam) — primary bootstrap
|
||||
// "/dns4/boot-eu1.blackwall.network/udp/4001/quic-v1/p2p/<PEER_ID>",
|
||||
// US-East (New York) — secondary bootstrap
|
||||
// "/dns4/boot-us1.blackwall.network/udp/4001/quic-v1/p2p/<PEER_ID>",
|
||||
// AP-South (Singapore) — tertiary bootstrap
|
||||
// "/dns4/boot-ap1.blackwall.network/udp/4001/quic-v1/p2p/<PEER_ID>",
|
||||
];
|
||||
|
||||
/// Connect to bootstrap nodes (default + user-configured) and initiate
|
||||
/// Kademlia bootstrap for full DHT peer discovery.
|
||||
///
|
||||
/// Bootstrap nodes are specified as multiaddresses in the config file.
|
||||
/// Each address must include a `/p2p/<peer-id>` component.
|
||||
pub fn connect_bootstrap_nodes(
|
||||
swarm: &mut Swarm<HiveMindBehaviour>,
|
||||
config: &HiveMindConfig,
|
||||
local_peer_id: &PeerId,
|
||||
) -> anyhow::Result<Vec<PeerId>> {
|
||||
let mut seed_peers = Vec::new();
|
||||
|
||||
// --- 1. Built-in (hardcoded) bootstrap nodes ---
|
||||
if config.bootstrap.use_default_nodes {
|
||||
for addr_str in DEFAULT_BOOTSTRAP_NODES {
|
||||
match try_add_bootstrap(swarm, addr_str, local_peer_id) {
|
||||
Ok(Some(pid)) => seed_peers.push(pid),
|
||||
Ok(None) => {} // self-referencing, skipped
|
||||
Err(e) => {
|
||||
warn!(
|
||||
addr = addr_str,
|
||||
error = %e,
|
||||
"Failed to parse default bootstrap node — skipping"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- 2. User-configured bootstrap nodes ---
|
||||
for node_addr_str in &config.bootstrap.nodes {
|
||||
match try_add_bootstrap(swarm, node_addr_str, local_peer_id) {
|
||||
Ok(Some(pid)) => seed_peers.push(pid),
|
||||
Ok(None) => {}
|
||||
Err(e) => {
|
||||
warn!(
|
||||
addr = node_addr_str,
|
||||
error = %e,
|
||||
"Failed to parse bootstrap node address — skipping"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !seed_peers.is_empty() {
|
||||
// Initiate Kademlia bootstrap to discover more peers
|
||||
swarm
|
||||
.behaviour_mut()
|
||||
.kademlia
|
||||
.bootstrap()
|
||||
.map_err(|e| anyhow::anyhow!("Kademlia bootstrap failed: {e:?}"))?;
|
||||
info!(count = seed_peers.len(), "Bootstrap initiated with known peers");
|
||||
} else {
|
||||
info!("No bootstrap nodes configured — relying on mDNS discovery");
|
||||
}
|
||||
|
||||
Ok(seed_peers)
|
||||
}
|
||||
|
||||
/// Try to add a single bootstrap node. Returns Ok(Some(PeerId)) if added,
|
||||
/// Ok(None) if skipped (self-referencing), Err on parse failure.
|
||||
fn try_add_bootstrap(
|
||||
swarm: &mut Swarm<HiveMindBehaviour>,
|
||||
addr_str: &str,
|
||||
local_peer_id: &PeerId,
|
||||
) -> anyhow::Result<Option<PeerId>> {
|
||||
let (peer_id, addr) = parse_bootstrap_addr(addr_str)?;
|
||||
|
||||
// SECURITY: Reject self-referencing bootstrap entries
|
||||
if peer_id == *local_peer_id {
|
||||
warn!("Skipping bootstrap node that references self");
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
// SECURITY: Reject loopback/unspecified addresses
|
||||
if !is_routable_addr(&addr) {
|
||||
warn!(%addr, "Skipping bootstrap node with non-routable address");
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
swarm
|
||||
.behaviour_mut()
|
||||
.kademlia
|
||||
.add_address(&peer_id, addr.clone());
|
||||
swarm
|
||||
.behaviour_mut()
|
||||
.gossipsub
|
||||
.add_explicit_peer(&peer_id);
|
||||
|
||||
info!(%peer_id, %addr, "Added bootstrap node");
|
||||
Ok(Some(peer_id))
|
||||
}
|
||||
|
||||
/// Parse a multiaddress string that includes a `/p2p/<peer-id>` suffix.
|
||||
///
|
||||
/// Example: `/ip4/104.131.131.82/udp/4001/quic-v1/p2p/QmPeer...`
|
||||
fn parse_bootstrap_addr(addr_str: &str) -> anyhow::Result<(PeerId, Multiaddr)> {
|
||||
let addr: Multiaddr = addr_str
|
||||
.parse()
|
||||
.context("Invalid multiaddress format")?;
|
||||
|
||||
// Extract PeerId from the /p2p/ component
|
||||
let peer_id = addr
|
||||
.iter()
|
||||
.find_map(|proto| {
|
||||
if let libp2p::multiaddr::Protocol::P2p(peer_id) = proto {
|
||||
Some(peer_id)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.context("Bootstrap address must include /p2p/<peer-id> component")?;
|
||||
|
||||
// Strip the /p2p/ component from the address for Kademlia
|
||||
let transport_addr: Multiaddr = addr
|
||||
.iter()
|
||||
.filter(|proto| !matches!(proto, libp2p::multiaddr::Protocol::P2p(_)))
|
||||
.collect();
|
||||
|
||||
Ok((peer_id, transport_addr))
|
||||
}
|
||||
|
||||
/// Handle mDNS discovery events — add discovered peers to both
|
||||
/// Kademlia and GossipSub.
|
||||
pub fn handle_mdns_discovered(
|
||||
swarm: &mut Swarm<HiveMindBehaviour>,
|
||||
peers: Vec<(PeerId, Multiaddr)>,
|
||||
local_peer_id: &PeerId,
|
||||
) {
|
||||
for (peer_id, addr) in peers {
|
||||
// SECURITY: Reject self-referencing entries
|
||||
if peer_id == *local_peer_id {
|
||||
continue;
|
||||
}
|
||||
|
||||
// SECURITY: Reject loopback/unspecified addresses
|
||||
if !is_routable_addr(&addr) {
|
||||
warn!(%peer_id, %addr, "mDNS: skipping non-routable address");
|
||||
continue;
|
||||
}
|
||||
|
||||
swarm
|
||||
.behaviour_mut()
|
||||
.kademlia
|
||||
.add_address(&peer_id, addr.clone());
|
||||
swarm
|
||||
.behaviour_mut()
|
||||
.gossipsub
|
||||
.add_explicit_peer(&peer_id);
|
||||
|
||||
info!(%peer_id, %addr, "mDNS: discovered local peer");
|
||||
}
|
||||
}
|
||||
|
||||
/// Handle mDNS expiry events — remove expired peers from GossipSub.
|
||||
pub fn handle_mdns_expired(
|
||||
swarm: &mut Swarm<HiveMindBehaviour>,
|
||||
peers: Vec<(PeerId, Multiaddr)>,
|
||||
) {
|
||||
for (peer_id, _addr) in peers {
|
||||
swarm
|
||||
.behaviour_mut()
|
||||
.gossipsub
|
||||
.remove_explicit_peer(&peer_id);
|
||||
|
||||
info!(%peer_id, "mDNS: peer expired");
|
||||
}
|
||||
}
|
||||
119
hivemind/src/config.rs
Executable file
119
hivemind/src/config.rs
Executable file
|
|
@ -0,0 +1,119 @@
|
|||
/// HiveMind configuration.
|
||||
use serde::Deserialize;
|
||||
use std::path::Path;
|
||||
|
||||
/// Node operating mode.
|
||||
///
|
||||
/// - `Full` — default: runs all modules (reputation, consensus, FL, metrics bridge).
|
||||
/// - `Bootstrap` — lightweight relay: only Kademlia + GossipSub forwarding.
|
||||
/// Designed for $5/mo VPS that hold tens of thousands of connections.
|
||||
/// No DPI, XDP, AI, ZKP, or federated learning.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Default)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum NodeMode {
|
||||
#[default]
|
||||
Full,
|
||||
Bootstrap,
|
||||
}
|
||||
|
||||
/// Top-level configuration for the HiveMind daemon.
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub struct HiveMindConfig {
|
||||
/// Node operating mode (full | bootstrap).
|
||||
#[serde(default)]
|
||||
pub mode: NodeMode,
|
||||
/// Explicit path to the identity key file.
|
||||
/// If omitted, defaults to ~/.blackwall/identity.key (or /etc/blackwall/identity.key as root).
|
||||
#[serde(default)]
|
||||
pub identity_key_path: Option<String>,
|
||||
/// Network configuration.
|
||||
#[serde(default)]
|
||||
pub network: NetworkConfig,
|
||||
/// Bootstrap configuration.
|
||||
#[serde(default)]
|
||||
pub bootstrap: BootstrapConfig,
|
||||
}
|
||||
|
||||
/// Network-level settings.
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub struct NetworkConfig {
|
||||
/// Listen address for QUIC transport (e.g., "/ip4/0.0.0.0/udp/4001/quic-v1").
|
||||
#[serde(default = "default_listen_addr")]
|
||||
pub listen_addr: String,
|
||||
/// Maximum GossipSub message size in bytes.
|
||||
#[serde(default = "default_max_message_size")]
|
||||
pub max_message_size: usize,
|
||||
/// GossipSub heartbeat interval in seconds.
|
||||
#[serde(default = "default_heartbeat_secs")]
|
||||
pub heartbeat_secs: u64,
|
||||
/// Idle connection timeout in seconds.
|
||||
#[serde(default = "default_idle_timeout_secs")]
|
||||
pub idle_timeout_secs: u64,
|
||||
}
|
||||
|
||||
/// Bootstrap node configuration.
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub struct BootstrapConfig {
|
||||
/// List of additional user-specified bootstrap multiaddresses.
|
||||
#[serde(default)]
|
||||
pub nodes: Vec<String>,
|
||||
/// Use built-in (hardcoded) bootstrap nodes. Default: true.
|
||||
/// Set to false only for isolated/private meshes.
|
||||
#[serde(default = "default_use_default_nodes")]
|
||||
pub use_default_nodes: bool,
|
||||
/// Enable mDNS for local peer discovery.
|
||||
#[serde(default = "default_mdns_enabled")]
|
||||
pub mdns_enabled: bool,
|
||||
}
|
||||
|
||||
impl Default for NetworkConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
listen_addr: default_listen_addr(),
|
||||
max_message_size: default_max_message_size(),
|
||||
heartbeat_secs: default_heartbeat_secs(),
|
||||
idle_timeout_secs: default_idle_timeout_secs(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for BootstrapConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
nodes: Vec::new(),
|
||||
use_default_nodes: default_use_default_nodes(),
|
||||
mdns_enabled: default_mdns_enabled(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn default_listen_addr() -> String {
|
||||
"/ip4/0.0.0.0/udp/4001/quic-v1".to_string()
|
||||
}
|
||||
|
||||
fn default_max_message_size() -> usize {
|
||||
common::hivemind::MAX_MESSAGE_SIZE
|
||||
}
|
||||
|
||||
fn default_heartbeat_secs() -> u64 {
|
||||
common::hivemind::GOSSIPSUB_HEARTBEAT_SECS
|
||||
}
|
||||
|
||||
fn default_idle_timeout_secs() -> u64 {
|
||||
60
|
||||
}
|
||||
|
||||
fn default_mdns_enabled() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
fn default_use_default_nodes() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
/// Load HiveMind configuration from a TOML file.
|
||||
pub fn load_config(path: &Path) -> anyhow::Result<HiveMindConfig> {
|
||||
let content = std::fs::read_to_string(path)?;
|
||||
let config: HiveMindConfig = toml::from_str(&content)?;
|
||||
Ok(config)
|
||||
}
|
||||
252
hivemind/src/consensus.rs
Executable file
252
hivemind/src/consensus.rs
Executable file
|
|
@ -0,0 +1,252 @@
|
|||
/// Cross-validation of IoCs through N independent peers.
|
||||
///
|
||||
/// A single peer's IoC report is never trusted. The consensus module
|
||||
/// tracks pending IoCs and requires at least `CROSS_VALIDATION_THRESHOLD`
|
||||
/// independent peer confirmations before an IoC is accepted into the
|
||||
/// local threat database.
|
||||
use common::hivemind::{self, IoC};
|
||||
use std::collections::HashMap;
|
||||
use tracing::{debug, info};
|
||||
|
||||
/// Unique key for deduplicating IoC submissions.
|
||||
///
|
||||
/// Two IoCs are considered equivalent if they share the same type, IP,
|
||||
/// and JA4 fingerprint.
|
||||
#[derive(Clone, Debug, Hash, PartialEq, Eq)]
|
||||
struct IoCKey {
|
||||
ioc_type: u8,
|
||||
ip: u32,
|
||||
ja4: Option<String>,
|
||||
}
|
||||
|
||||
impl From<&IoC> for IoCKey {
|
||||
fn from(ioc: &IoC) -> Self {
|
||||
Self {
|
||||
ioc_type: ioc.ioc_type,
|
||||
ip: ioc.ip,
|
||||
ja4: ioc.ja4.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A pending IoC awaiting cross-validation from multiple peers.
|
||||
#[derive(Clone, Debug)]
|
||||
struct PendingIoC {
|
||||
/// The IoC being validated.
|
||||
ioc: IoC,
|
||||
/// Set of peer pubkeys that confirmed this IoC (Ed25519, 32 bytes).
|
||||
confirmations: Vec<[u8; 32]>,
|
||||
/// Unix timestamp when this pending entry was created.
|
||||
created_at: u64,
|
||||
}
|
||||
|
||||
/// Result of submitting a peer's IoC confirmation.
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub enum ConsensusResult {
|
||||
/// IoC accepted — threshold reached. Contains final confirmation count.
|
||||
Accepted(usize),
|
||||
/// IoC recorded but threshold not yet met. Contains current count.
|
||||
Pending(usize),
|
||||
/// Duplicate confirmation from the same peer — ignored.
|
||||
DuplicatePeer,
|
||||
/// The IoC expired before reaching consensus.
|
||||
Expired,
|
||||
}
|
||||
|
||||
/// Manages cross-validation of IoCs across independent peers.
|
||||
pub struct ConsensusEngine {
|
||||
/// Pending IoCs keyed by their dedup identity.
|
||||
pending: HashMap<IoCKey, PendingIoC>,
|
||||
/// IoCs that reached consensus (for querying accepted threats).
|
||||
accepted: Vec<IoC>,
|
||||
}
|
||||
|
||||
impl Default for ConsensusEngine {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl ConsensusEngine {
|
||||
/// Create a new consensus engine.
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
pending: HashMap::new(),
|
||||
accepted: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Submit a peer's IoC report for cross-validation.
|
||||
///
|
||||
/// Returns the consensus result: whether threshold was met, pending, or duplicate.
|
||||
pub fn submit_ioc(
|
||||
&mut self,
|
||||
ioc: &IoC,
|
||||
reporter_pubkey: &[u8; 32],
|
||||
) -> ConsensusResult {
|
||||
let key = IoCKey::from(ioc);
|
||||
let now = now_secs();
|
||||
|
||||
// Check if this IoC is already pending
|
||||
if let Some(pending) = self.pending.get_mut(&key) {
|
||||
// Check expiry
|
||||
if now.saturating_sub(pending.created_at) > hivemind::CONSENSUS_TIMEOUT_SECS {
|
||||
debug!("Pending IoC expired — removing");
|
||||
self.pending.remove(&key);
|
||||
return ConsensusResult::Expired;
|
||||
}
|
||||
|
||||
// Check for duplicate peer confirmation
|
||||
if pending.confirmations.iter().any(|pk| pk == reporter_pubkey) {
|
||||
debug!("Duplicate confirmation from same peer — ignoring");
|
||||
return ConsensusResult::DuplicatePeer;
|
||||
}
|
||||
|
||||
pending.confirmations.push(*reporter_pubkey);
|
||||
let count = pending.confirmations.len();
|
||||
|
||||
if count >= hivemind::CROSS_VALIDATION_THRESHOLD {
|
||||
// Consensus reached — move to accepted
|
||||
let mut accepted_ioc = pending.ioc.clone();
|
||||
accepted_ioc.confirmations = count as u32;
|
||||
info!(
|
||||
count,
|
||||
ioc_type = accepted_ioc.ioc_type,
|
||||
"IoC reached consensus — accepted"
|
||||
);
|
||||
self.accepted.push(accepted_ioc);
|
||||
self.pending.remove(&key);
|
||||
return ConsensusResult::Accepted(count);
|
||||
}
|
||||
|
||||
debug!(count, threshold = hivemind::CROSS_VALIDATION_THRESHOLD, "IoC pending");
|
||||
ConsensusResult::Pending(count)
|
||||
} else {
|
||||
// First report of this IoC
|
||||
let pending = PendingIoC {
|
||||
ioc: ioc.clone(),
|
||||
confirmations: vec![*reporter_pubkey],
|
||||
created_at: now,
|
||||
};
|
||||
self.pending.insert(key, pending);
|
||||
debug!("New IoC submitted — awaiting cross-validation");
|
||||
ConsensusResult::Pending(1)
|
||||
}
|
||||
}
|
||||
|
||||
/// Drain all newly accepted IoCs. Returns and clears the accepted list.
|
||||
pub fn drain_accepted(&mut self) -> Vec<IoC> {
|
||||
std::mem::take(&mut self.accepted)
|
||||
}
|
||||
|
||||
/// Evict expired pending IoCs. Returns the number removed.
|
||||
pub fn evict_expired(&mut self) -> usize {
|
||||
let now = now_secs();
|
||||
let before = self.pending.len();
|
||||
self.pending.retain(|_, pending| {
|
||||
now.saturating_sub(pending.created_at) <= hivemind::CONSENSUS_TIMEOUT_SECS
|
||||
});
|
||||
let removed = before - self.pending.len();
|
||||
if removed > 0 {
|
||||
info!(removed, "Evicted expired pending IoCs");
|
||||
}
|
||||
removed
|
||||
}
|
||||
|
||||
/// Number of IoCs currently awaiting consensus.
|
||||
pub fn pending_count(&self) -> usize {
|
||||
self.pending.len()
|
||||
}
|
||||
}
|
||||
|
||||
fn now_secs() -> u64 {
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn make_ioc() -> IoC {
|
||||
IoC {
|
||||
ioc_type: 0,
|
||||
severity: 3,
|
||||
ip: 0xC0A80001, // 192.168.0.1
|
||||
ja4: Some("t13d1516h2_8daaf6152771_e5627efa2ab1".to_string()),
|
||||
entropy_score: Some(7500),
|
||||
description: "Test malicious IP".to_string(),
|
||||
first_seen: 1700000000,
|
||||
confirmations: 0,
|
||||
zkp_proof: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn peer_key(id: u8) -> [u8; 32] {
|
||||
let mut key = [0u8; 32];
|
||||
key[0] = id;
|
||||
key
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn single_report_stays_pending() {
|
||||
let mut engine = ConsensusEngine::new();
|
||||
let ioc = make_ioc();
|
||||
let result = engine.submit_ioc(&ioc, &peer_key(1));
|
||||
assert_eq!(result, ConsensusResult::Pending(1));
|
||||
assert_eq!(engine.pending_count(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn duplicate_peer_ignored() {
|
||||
let mut engine = ConsensusEngine::new();
|
||||
let ioc = make_ioc();
|
||||
engine.submit_ioc(&ioc, &peer_key(1));
|
||||
let result = engine.submit_ioc(&ioc, &peer_key(1));
|
||||
assert_eq!(result, ConsensusResult::DuplicatePeer);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn consensus_reached_at_threshold() {
|
||||
let mut engine = ConsensusEngine::new();
|
||||
let ioc = make_ioc();
|
||||
|
||||
for i in 1..hivemind::CROSS_VALIDATION_THRESHOLD {
|
||||
let result = engine.submit_ioc(&ioc, &peer_key(i as u8));
|
||||
assert_eq!(result, ConsensusResult::Pending(i));
|
||||
}
|
||||
|
||||
let result = engine.submit_ioc(
|
||||
&ioc,
|
||||
&peer_key(hivemind::CROSS_VALIDATION_THRESHOLD as u8),
|
||||
);
|
||||
assert_eq!(
|
||||
result,
|
||||
ConsensusResult::Accepted(hivemind::CROSS_VALIDATION_THRESHOLD)
|
||||
);
|
||||
assert_eq!(engine.pending_count(), 0);
|
||||
|
||||
let accepted = engine.drain_accepted();
|
||||
assert_eq!(accepted.len(), 1);
|
||||
assert_eq!(
|
||||
accepted[0].confirmations,
|
||||
hivemind::CROSS_VALIDATION_THRESHOLD as u32
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn different_iocs_tracked_separately() {
|
||||
let mut engine = ConsensusEngine::new();
|
||||
|
||||
let mut ioc1 = make_ioc();
|
||||
ioc1.ip = 1;
|
||||
let mut ioc2 = make_ioc();
|
||||
ioc2.ip = 2;
|
||||
|
||||
engine.submit_ioc(&ioc1, &peer_key(1));
|
||||
engine.submit_ioc(&ioc2, &peer_key(1));
|
||||
assert_eq!(engine.pending_count(), 2);
|
||||
}
|
||||
}
|
||||
439
hivemind/src/crypto/fhe.rs
Executable file
439
hivemind/src/crypto/fhe.rs
Executable file
|
|
@ -0,0 +1,439 @@
|
|||
/// FHE (Fully Homomorphic Encryption) — gradient privacy via AES-256-GCM.
|
||||
///
|
||||
/// # Privacy Invariant
|
||||
/// Raw gradients NEVER leave the node. Only ciphertext is transmitted.
|
||||
///
|
||||
/// # Implementation
|
||||
/// - **v0 (legacy stub)**: `SFHE` prefix + raw f32 LE bytes (no real encryption).
|
||||
/// - **v1 (encrypted)**: `RFHE` prefix + AES-256-GCM encrypted payload.
|
||||
///
|
||||
/// True homomorphic operations (add/multiply on ciphertext) require `tfhe-rs`
|
||||
/// and are feature-gated for Phase 2+. Current encryption provides
|
||||
/// confidentiality at rest and in transit but is NOT homomorphic —
|
||||
/// the aggregator must decrypt before aggregating.
|
||||
///
|
||||
/// # Naming Convention
|
||||
/// The primary type is `GradientCryptoCtx` (accurate to current implementation).
|
||||
/// `FheContext` is a type alias preserved for backward compatibility and will
|
||||
/// become the real FHE wrapper when `tfhe-rs` is integrated in Phase 2+.
|
||||
use ring::aead::{self, Aad, BoundKey, Nonce, NonceSequence, NONCE_LEN};
|
||||
use ring::rand::{SecureRandom, SystemRandom};
|
||||
use tracing::{debug, warn};
|
||||
|
||||
/// Magic bytes identifying a v0 stub (unencrypted) payload.
|
||||
const STUB_FHE_MAGIC: &[u8; 4] = b"SFHE";
|
||||
|
||||
/// Magic bytes identifying a v1 AES-256-GCM encrypted payload.
|
||||
const REAL_FHE_MAGIC: &[u8; 4] = b"RFHE";
|
||||
|
||||
/// Nonce size for AES-256-GCM (96 bits).
|
||||
const NONCE_SIZE: usize = NONCE_LEN;
|
||||
|
||||
/// Overhead: magic(4) + nonce(12) + GCM tag(16) = 32 bytes.
|
||||
const ENCRYPTION_OVERHEAD: usize = 4 + NONCE_SIZE + 16;
|
||||
|
||||
/// Single-use nonce for AES-256-GCM sealing operations.
|
||||
struct OneNonceSequence(Option<aead::Nonce>);
|
||||
|
||||
impl OneNonceSequence {
|
||||
fn new(nonce_bytes: [u8; NONCE_SIZE]) -> Self {
|
||||
Self(Some(aead::Nonce::assume_unique_for_key(nonce_bytes)))
|
||||
}
|
||||
}
|
||||
|
||||
impl NonceSequence for OneNonceSequence {
|
||||
fn advance(&mut self) -> Result<Nonce, ring::error::Unspecified> {
|
||||
self.0.take().ok_or(ring::error::Unspecified)
|
||||
}
|
||||
}
|
||||
|
||||
/// Gradient encryption context using AES-256-GCM.
|
||||
///
|
||||
/// Provides confidentiality for gradient vectors transmitted over GossipSub.
|
||||
/// NOT truly homomorphic — aggregator must decrypt before aggregating.
|
||||
/// Will be replaced by real FHE (`tfhe-rs`) in Phase 2+.
|
||||
pub struct GradientCryptoCtx {
|
||||
/// Raw AES-256-GCM key material (32 bytes).
|
||||
key_bytes: Vec<u8>,
|
||||
/// Whether this context has been initialized with keys.
|
||||
initialized: bool,
|
||||
}
|
||||
|
||||
/// Backward-compatible alias. Will point to a real FHE wrapper in Phase 2+.
|
||||
pub type FheContext = GradientCryptoCtx;
|
||||
|
||||
impl Default for GradientCryptoCtx {
|
||||
fn default() -> Self {
|
||||
Self::new_encrypted().expect("GradientCryptoCtx initialization failed")
|
||||
}
|
||||
}
|
||||
|
||||
impl GradientCryptoCtx {
|
||||
/// Create a new context with a fresh AES-256-GCM key.
|
||||
///
|
||||
/// Generates a random 256-bit key using the system CSPRNG.
|
||||
pub fn new_encrypted() -> Result<Self, FheError> {
|
||||
let rng = SystemRandom::new();
|
||||
let mut key_bytes = vec![0u8; 32];
|
||||
rng.fill(&mut key_bytes)
|
||||
.map_err(|_| FheError::KeyGenerationFailed)?;
|
||||
|
||||
debug!("FHE context initialized (AES-256-GCM)");
|
||||
Ok(Self {
|
||||
key_bytes,
|
||||
initialized: true,
|
||||
})
|
||||
}
|
||||
|
||||
/// Create a legacy stub context (no real encryption).
|
||||
///
|
||||
/// For backward compatibility only. New code should use `new_encrypted()`.
|
||||
pub fn new() -> Self {
|
||||
debug!("FHE context initialized (stub — no real encryption)");
|
||||
Self {
|
||||
key_bytes: Vec::new(),
|
||||
initialized: true,
|
||||
}
|
||||
}
|
||||
|
||||
/// Create FHE context from existing key material.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `key_bytes` — 32-byte AES-256-GCM key
|
||||
pub fn from_key(key_bytes: &[u8]) -> Result<Self, FheError> {
|
||||
if key_bytes.len() != 32 {
|
||||
return Err(FheError::InvalidPayload);
|
||||
}
|
||||
Ok(Self {
|
||||
key_bytes: key_bytes.to_vec(),
|
||||
initialized: true,
|
||||
})
|
||||
}
|
||||
|
||||
/// Encrypt gradient vector for safe transmission over GossipSub.
|
||||
///
|
||||
/// # Privacy Contract
|
||||
/// The returned bytes are AES-256-GCM encrypted — not reversible
|
||||
/// without the symmetric key.
|
||||
///
|
||||
/// # Format
|
||||
/// `RFHE(4B) || nonce(12B) || ciphertext+tag`
|
||||
///
|
||||
/// Falls back to stub format if no key material is available.
|
||||
pub fn encrypt_gradients(&self, gradients: &[f32]) -> Result<Vec<u8>, FheError> {
|
||||
if !self.initialized {
|
||||
return Err(FheError::Uninitialized);
|
||||
}
|
||||
|
||||
// Stub mode — no key material
|
||||
if self.key_bytes.is_empty() {
|
||||
return self.encrypt_stub(gradients);
|
||||
}
|
||||
|
||||
// Serialize gradients to raw bytes
|
||||
let mut plaintext = Vec::with_capacity(gradients.len() * 4);
|
||||
for &g in gradients {
|
||||
plaintext.extend_from_slice(&g.to_le_bytes());
|
||||
}
|
||||
|
||||
// Generate random nonce
|
||||
let rng = SystemRandom::new();
|
||||
let mut nonce_bytes = [0u8; NONCE_SIZE];
|
||||
rng.fill(&mut nonce_bytes)
|
||||
.map_err(|_| FheError::EncryptionFailed)?;
|
||||
|
||||
// Create sealing key
|
||||
let unbound_key = aead::UnboundKey::new(&aead::AES_256_GCM, &self.key_bytes)
|
||||
.map_err(|_| FheError::EncryptionFailed)?;
|
||||
let nonce_seq = OneNonceSequence::new(nonce_bytes);
|
||||
let mut sealing_key = aead::SealingKey::new(unbound_key, nonce_seq);
|
||||
|
||||
// Encrypt in-place (appends GCM tag)
|
||||
sealing_key
|
||||
.seal_in_place_append_tag(Aad::empty(), &mut plaintext)
|
||||
.map_err(|_| FheError::EncryptionFailed)?;
|
||||
|
||||
// Build output: RFHE || nonce || ciphertext+tag
|
||||
let mut payload = Vec::with_capacity(ENCRYPTION_OVERHEAD + plaintext.len());
|
||||
payload.extend_from_slice(REAL_FHE_MAGIC);
|
||||
payload.extend_from_slice(&nonce_bytes);
|
||||
payload.extend_from_slice(&plaintext);
|
||||
|
||||
debug!(
|
||||
gradient_count = gradients.len(),
|
||||
payload_size = payload.len(),
|
||||
"gradients encrypted (AES-256-GCM)"
|
||||
);
|
||||
Ok(payload)
|
||||
}
|
||||
|
||||
/// Legacy stub encryption (no real crypto).
|
||||
fn encrypt_stub(&self, gradients: &[f32]) -> Result<Vec<u8>, FheError> {
|
||||
let mut payload = Vec::with_capacity(4 + gradients.len() * 4);
|
||||
payload.extend_from_slice(STUB_FHE_MAGIC);
|
||||
for &g in gradients {
|
||||
payload.extend_from_slice(&g.to_le_bytes());
|
||||
}
|
||||
debug!(
|
||||
gradient_count = gradients.len(),
|
||||
"gradients serialized (stub — no encryption)"
|
||||
);
|
||||
Ok(payload)
|
||||
}
|
||||
|
||||
/// Decrypt a gradient payload received from GossipSub.
|
||||
///
|
||||
/// Supports both RFHE (encrypted) and SFHE (legacy stub) payloads.
|
||||
pub fn decrypt_gradients(&self, payload: &[u8]) -> Result<Vec<f32>, FheError> {
|
||||
if !self.initialized {
|
||||
return Err(FheError::Uninitialized);
|
||||
}
|
||||
|
||||
if payload.len() < 4 {
|
||||
return Err(FheError::InvalidPayload);
|
||||
}
|
||||
|
||||
match &payload[..4] {
|
||||
b"RFHE" => self.decrypt_real(payload),
|
||||
b"SFHE" => self.decrypt_stub(payload),
|
||||
_ => {
|
||||
warn!("unknown FHE payload format");
|
||||
Err(FheError::InvalidPayload)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Decrypt an AES-256-GCM encrypted payload.
|
||||
fn decrypt_real(&self, payload: &[u8]) -> Result<Vec<f32>, FheError> {
|
||||
if self.key_bytes.is_empty() {
|
||||
return Err(FheError::Uninitialized);
|
||||
}
|
||||
|
||||
// Minimum: magic(4) + nonce(12) + tag(16) = 32 bytes
|
||||
if payload.len() < ENCRYPTION_OVERHEAD {
|
||||
return Err(FheError::InvalidPayload);
|
||||
}
|
||||
|
||||
let nonce_bytes: [u8; NONCE_SIZE] = payload[4..4 + NONCE_SIZE]
|
||||
.try_into()
|
||||
.map_err(|_| FheError::InvalidPayload)?;
|
||||
let mut ciphertext = payload[4 + NONCE_SIZE..].to_vec();
|
||||
|
||||
let unbound_key = aead::UnboundKey::new(&aead::AES_256_GCM, &self.key_bytes)
|
||||
.map_err(|_| FheError::DecryptionFailed)?;
|
||||
let nonce_seq = OneNonceSequence::new(nonce_bytes);
|
||||
let mut opening_key = aead::OpeningKey::new(unbound_key, nonce_seq);
|
||||
|
||||
let plaintext = opening_key
|
||||
.open_in_place(Aad::empty(), &mut ciphertext)
|
||||
.map_err(|_| FheError::DecryptionFailed)?;
|
||||
|
||||
if !plaintext.len().is_multiple_of(4) {
|
||||
return Err(FheError::InvalidPayload);
|
||||
}
|
||||
|
||||
let gradients: Vec<f32> = plaintext
|
||||
.chunks_exact(4)
|
||||
.map(|chunk| {
|
||||
let bytes: [u8; 4] = chunk.try_into().expect("chunk is 4 bytes");
|
||||
f32::from_le_bytes(bytes)
|
||||
})
|
||||
.collect();
|
||||
|
||||
debug!(gradient_count = gradients.len(), "gradients decrypted (AES-256-GCM)");
|
||||
Ok(gradients)
|
||||
}
|
||||
|
||||
/// Decrypt a legacy stub payload (just deserialization, no crypto).
|
||||
fn decrypt_stub(&self, payload: &[u8]) -> Result<Vec<f32>, FheError> {
|
||||
let data = &payload[4..];
|
||||
if !data.len().is_multiple_of(4) {
|
||||
return Err(FheError::InvalidPayload);
|
||||
}
|
||||
|
||||
let gradients: Vec<f32> = data
|
||||
.chunks_exact(4)
|
||||
.map(|chunk| {
|
||||
let bytes: [u8; 4] = chunk.try_into().expect("chunk is 4 bytes");
|
||||
f32::from_le_bytes(bytes)
|
||||
})
|
||||
.collect();
|
||||
|
||||
debug!(gradient_count = gradients.len(), "gradients deserialized (stub)");
|
||||
Ok(gradients)
|
||||
}
|
||||
|
||||
/// Check if this is a stub (unencrypted) implementation.
|
||||
pub fn is_stub(&self) -> bool {
|
||||
self.key_bytes.is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
/// Errors from FHE operations.
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub enum FheError {
|
||||
/// FHE context not initialized (no keys generated).
|
||||
Uninitialized,
|
||||
/// Payload format is invalid or corrupted.
|
||||
InvalidPayload,
|
||||
/// Key generation failed (CSPRNG error).
|
||||
KeyGenerationFailed,
|
||||
/// AES-256-GCM encryption failed.
|
||||
EncryptionFailed,
|
||||
/// AES-256-GCM decryption failed (tampered or wrong key).
|
||||
DecryptionFailed,
|
||||
}
|
||||
|
||||
impl std::fmt::Display for FheError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
FheError::Uninitialized => write!(f, "FHE context not initialized"),
|
||||
FheError::InvalidPayload => write!(f, "invalid FHE payload format"),
|
||||
FheError::KeyGenerationFailed => write!(f, "FHE key generation failed"),
|
||||
FheError::EncryptionFailed => write!(f, "AES-256-GCM encryption failed"),
|
||||
FheError::DecryptionFailed => write!(f, "AES-256-GCM decryption failed"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for FheError {}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn encrypted_roundtrip() {
|
||||
let ctx = FheContext::new_encrypted().expect("init");
|
||||
let gradients = vec![1.0_f32, -0.5, 0.0, 3.14, -2.718];
|
||||
let encrypted = ctx.encrypt_gradients(&gradients).expect("encrypt");
|
||||
let decrypted = ctx.decrypt_gradients(&encrypted).expect("decrypt");
|
||||
assert_eq!(gradients, decrypted);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn encrypted_has_rfhe_prefix() {
|
||||
let ctx = FheContext::new_encrypted().expect("init");
|
||||
let encrypted = ctx.encrypt_gradients(&[1.0, 2.0]).expect("encrypt");
|
||||
assert_eq!(&encrypted[..4], REAL_FHE_MAGIC);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn encrypted_payload_larger_than_stub() {
|
||||
let ctx_real = FheContext::new_encrypted().expect("init");
|
||||
let ctx_stub = FheContext::new();
|
||||
let grads = vec![1.0_f32; 10];
|
||||
let real = ctx_real.encrypt_gradients(&grads).expect("encrypt");
|
||||
let stub = ctx_stub.encrypt_gradients(&grads).expect("encrypt");
|
||||
// Real encryption adds nonce(12) + tag(16) overhead
|
||||
assert!(real.len() > stub.len());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn wrong_key_fails_decryption() {
|
||||
let ctx1 = FheContext::new_encrypted().expect("init");
|
||||
let ctx2 = FheContext::new_encrypted().expect("init");
|
||||
let encrypted = ctx1.encrypt_gradients(&[1.0, 2.0]).expect("encrypt");
|
||||
assert_eq!(
|
||||
ctx2.decrypt_gradients(&encrypted),
|
||||
Err(FheError::DecryptionFailed),
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tampered_ciphertext_fails() {
|
||||
let ctx = FheContext::new_encrypted().expect("init");
|
||||
let mut encrypted = ctx.encrypt_gradients(&[1.0, 2.0]).expect("encrypt");
|
||||
// Tamper with the ciphertext (after nonce)
|
||||
let idx = 4 + NONCE_SIZE + 1;
|
||||
if idx < encrypted.len() {
|
||||
encrypted[idx] ^= 0xFF;
|
||||
}
|
||||
assert_eq!(
|
||||
ctx.decrypt_gradients(&encrypted),
|
||||
Err(FheError::DecryptionFailed),
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stub_roundtrip() {
|
||||
let ctx = FheContext::new();
|
||||
let gradients = vec![1.0_f32, -0.5, 0.0, 3.14, -2.718];
|
||||
let encrypted = ctx.encrypt_gradients(&gradients).expect("encrypt");
|
||||
let decrypted = ctx.decrypt_gradients(&encrypted).expect("decrypt");
|
||||
assert_eq!(gradients, decrypted);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stub_has_sfhe_prefix() {
|
||||
let ctx = FheContext::new();
|
||||
let encrypted = ctx.encrypt_gradients(&[1.0]).expect("encrypt");
|
||||
assert_eq!(&encrypted[..4], STUB_FHE_MAGIC);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_invalid_payload() {
|
||||
let ctx = FheContext::new_encrypted().expect("init");
|
||||
assert_eq!(
|
||||
ctx.decrypt_gradients(&[0xDE, 0xAD]),
|
||||
Err(FheError::InvalidPayload),
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_wrong_magic() {
|
||||
let ctx = FheContext::new_encrypted().expect("init");
|
||||
let bad = b"BADx\x00\x00\x80\x3f";
|
||||
assert_eq!(
|
||||
ctx.decrypt_gradients(bad),
|
||||
Err(FheError::InvalidPayload),
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_gradients_encrypted() {
|
||||
let ctx = FheContext::new_encrypted().expect("init");
|
||||
let encrypted = ctx.encrypt_gradients(&[]).expect("encrypt");
|
||||
let decrypted = ctx.decrypt_gradients(&encrypted).expect("decrypt");
|
||||
assert!(decrypted.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn is_stub_reports_correctly() {
|
||||
let stub = FheContext::new();
|
||||
assert!(stub.is_stub());
|
||||
|
||||
let real = FheContext::new_encrypted().expect("init");
|
||||
assert!(!real.is_stub());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn from_key_roundtrip() {
|
||||
let ctx1 = FheContext::new_encrypted().expect("init");
|
||||
let encrypted = ctx1.encrypt_gradients(&[42.0, -1.0]).expect("encrypt");
|
||||
|
||||
// Reconstruct context from same key material
|
||||
let ctx2 = FheContext::from_key(&ctx1.key_bytes).expect("from_key");
|
||||
let decrypted = ctx2.decrypt_gradients(&encrypted).expect("decrypt");
|
||||
assert_eq!(decrypted, vec![42.0, -1.0]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn real_ctx_can_read_stub_payload() {
|
||||
let stub = FheContext::new();
|
||||
let encrypted = stub.encrypt_gradients(&[1.0, 2.0]).expect("encrypt");
|
||||
|
||||
let real = FheContext::new_encrypted().expect("init");
|
||||
let decrypted = real.decrypt_gradients(&encrypted).expect("decrypt");
|
||||
assert_eq!(decrypted, vec![1.0, 2.0]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn different_nonces_produce_different_ciphertext() {
|
||||
let ctx = FheContext::new_encrypted().expect("init");
|
||||
let e1 = ctx.encrypt_gradients(&[1.0]).expect("encrypt");
|
||||
let e2 = ctx.encrypt_gradients(&[1.0]).expect("encrypt");
|
||||
// Different nonces → different ciphertext
|
||||
assert_ne!(e1, e2);
|
||||
}
|
||||
}
|
||||
187
hivemind/src/crypto/fhe_real.rs
Executable file
187
hivemind/src/crypto/fhe_real.rs
Executable file
|
|
@ -0,0 +1,187 @@
|
|||
//! Real Fully Homomorphic Encryption using TFHE-rs.
|
||||
//!
|
||||
//! Feature-gated behind `fhe-real`. Provides true homomorphic operations
|
||||
//! on encrypted gradient vectors — the aggregator can sum encrypted gradients
|
||||
//! WITHOUT decrypting them. This eliminates the trust requirement on the
|
||||
//! aggregator node in federated learning.
|
||||
//!
|
||||
//! # Architecture
|
||||
//! - Each node generates a `ClientKey` (private) and `ServerKey` (public).
|
||||
//! - Gradients are encrypted with `ClientKey` → `FheInt32` ciphertext.
|
||||
//! - The aggregator uses `ServerKey` to homomorphically add ciphertexts.
|
||||
//! - Only the originating node can decrypt with its `ClientKey`.
|
||||
//!
|
||||
//! # Performance
|
||||
//! TFHE operations are CPU-intensive. For 8GB VRAM systems:
|
||||
//! - Batch gradients into chunks of 64 before encryption
|
||||
//! - Use shortint parameters for efficiency
|
||||
//! - Aggregation is async to avoid blocking the event loop
|
||||
|
||||
use tfhe::prelude::*;
|
||||
use tfhe::{generate_keys, set_server_key, ClientKey, ConfigBuilder, FheInt32, ServerKey};
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
/// Real FHE context using TFHE-rs for homomorphic gradient operations.
|
||||
pub struct RealFheContext {
|
||||
client_key: ClientKey,
|
||||
server_key: ServerKey,
|
||||
}
|
||||
|
||||
/// Errors from real FHE operations.
|
||||
#[derive(Debug)]
|
||||
pub enum RealFheError {
|
||||
/// Key generation failed.
|
||||
KeyGenerationFailed(String),
|
||||
/// Encryption failed.
|
||||
EncryptionFailed(String),
|
||||
/// Decryption failed.
|
||||
DecryptionFailed(String),
|
||||
/// Homomorphic operation failed.
|
||||
HomomorphicOpFailed(String),
|
||||
}
|
||||
|
||||
impl std::fmt::Display for RealFheError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
Self::KeyGenerationFailed(e) => write!(f, "FHE key generation failed: {}", e),
|
||||
Self::EncryptionFailed(e) => write!(f, "FHE encryption failed: {}", e),
|
||||
Self::DecryptionFailed(e) => write!(f, "FHE decryption failed: {}", e),
|
||||
Self::HomomorphicOpFailed(e) => write!(f, "FHE homomorphic op failed: {}", e),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for RealFheError {}
|
||||
|
||||
impl RealFheContext {
|
||||
/// Generate fresh FHE keys (expensive — ~seconds on first call).
|
||||
///
|
||||
/// The `ServerKey` should be distributed to aggregator nodes.
|
||||
/// The `ClientKey` stays private on this node.
|
||||
pub fn new() -> Result<Self, RealFheError> {
|
||||
info!("generating TFHE keys (this may take a moment)...");
|
||||
let config = ConfigBuilder::default().build();
|
||||
let (client_key, server_key) = generate_keys(config);
|
||||
info!("TFHE keys generated — real FHE enabled");
|
||||
Ok(Self {
|
||||
client_key,
|
||||
server_key,
|
||||
})
|
||||
}
|
||||
|
||||
/// Get a reference to the server key (for distribution to aggregators).
|
||||
pub fn server_key(&self) -> &ServerKey {
|
||||
&self.server_key
|
||||
}
|
||||
|
||||
/// Encrypt gradient values as FHE ciphertexts.
|
||||
///
|
||||
/// Quantizes f32 gradients to i32 (×10000 for 4 decimal places precision)
|
||||
/// before encryption. Returns serialized ciphertexts.
|
||||
pub fn encrypt_gradients(&self, gradients: &[f32]) -> Result<Vec<Vec<u8>>, RealFheError> {
|
||||
let mut encrypted = Vec::with_capacity(gradients.len());
|
||||
for (i, &g) in gradients.iter().enumerate() {
|
||||
// Quantize: f32 → i32 with 4dp precision
|
||||
let quantized = (g * 10000.0) as i32;
|
||||
let ct = FheInt32::encrypt(quantized, &self.client_key);
|
||||
let bytes = bincode::serialize(&ct)
|
||||
.map_err(|e| RealFheError::EncryptionFailed(e.to_string()))?;
|
||||
encrypted.push(bytes);
|
||||
if i % 64 == 0 && i > 0 {
|
||||
debug!(progress = i, total = gradients.len(), "FHE encryption progress");
|
||||
}
|
||||
}
|
||||
debug!(count = gradients.len(), "gradients encrypted with TFHE");
|
||||
Ok(encrypted)
|
||||
}
|
||||
|
||||
/// Decrypt FHE ciphertexts back to gradient values.
|
||||
pub fn decrypt_gradients(&self, ciphertexts: &[Vec<u8>]) -> Result<Vec<f32>, RealFheError> {
|
||||
let mut gradients = Vec::with_capacity(ciphertexts.len());
|
||||
for ct_bytes in ciphertexts {
|
||||
let ct: FheInt32 = bincode::deserialize(ct_bytes)
|
||||
.map_err(|e| RealFheError::DecryptionFailed(e.to_string()))?;
|
||||
let quantized: i32 = ct.decrypt(&self.client_key);
|
||||
gradients.push(quantized as f32 / 10000.0);
|
||||
}
|
||||
debug!(count = gradients.len(), "gradients decrypted from TFHE");
|
||||
Ok(gradients)
|
||||
}
|
||||
|
||||
/// Homomorphically add two encrypted gradient vectors (element-wise).
|
||||
///
|
||||
/// This is the core FL aggregation operation — runs on the aggregator
|
||||
/// node WITHOUT access to the client key (plaintext never exposed).
|
||||
pub fn aggregate_encrypted(
|
||||
&self,
|
||||
a: &[Vec<u8>],
|
||||
b: &[Vec<u8>],
|
||||
) -> Result<Vec<Vec<u8>>, RealFheError> {
|
||||
if a.len() != b.len() {
|
||||
return Err(RealFheError::HomomorphicOpFailed(
|
||||
"gradient vector length mismatch".into(),
|
||||
));
|
||||
}
|
||||
|
||||
// Set server key for homomorphic operations
|
||||
set_server_key(self.server_key.clone());
|
||||
|
||||
let mut result = Vec::with_capacity(a.len());
|
||||
for (ct_a, ct_b) in a.iter().zip(b.iter()) {
|
||||
let a_ct: FheInt32 = bincode::deserialize(ct_a)
|
||||
.map_err(|e| RealFheError::HomomorphicOpFailed(e.to_string()))?;
|
||||
let b_ct: FheInt32 = bincode::deserialize(ct_b)
|
||||
.map_err(|e| RealFheError::HomomorphicOpFailed(e.to_string()))?;
|
||||
|
||||
// Homomorphic addition — no decryption needed!
|
||||
let sum = a_ct + b_ct;
|
||||
let bytes = bincode::serialize(&sum)
|
||||
.map_err(|e| RealFheError::HomomorphicOpFailed(e.to_string()))?;
|
||||
result.push(bytes);
|
||||
}
|
||||
debug!(count = a.len(), "encrypted gradients aggregated homomorphically");
|
||||
Ok(result)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn fhe_encrypt_decrypt_roundtrip() {
|
||||
let ctx = RealFheContext::new().expect("key gen");
|
||||
let gradients = vec![1.5f32, -0.25, 0.0, 3.1415];
|
||||
let encrypted = ctx.encrypt_gradients(&gradients).expect("encrypt");
|
||||
let decrypted = ctx.decrypt_gradients(&encrypted).expect("decrypt");
|
||||
|
||||
// Check within quantization tolerance (4dp = 0.0001)
|
||||
for (orig, dec) in gradients.iter().zip(decrypted.iter()) {
|
||||
assert!((orig - dec).abs() < 0.001, "mismatch: {} vs {}", orig, dec);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fhe_homomorphic_addition() {
|
||||
let ctx = RealFheContext::new().expect("key gen");
|
||||
|
||||
let a = vec![1.0f32, 2.0, 3.0];
|
||||
let b = vec![4.0f32, 5.0, 6.0];
|
||||
|
||||
let enc_a = ctx.encrypt_gradients(&a).expect("encrypt a");
|
||||
let enc_b = ctx.encrypt_gradients(&b).expect("encrypt b");
|
||||
|
||||
let enc_sum = ctx.aggregate_encrypted(&enc_a, &enc_b).expect("aggregate");
|
||||
let sum = ctx.decrypt_gradients(&enc_sum).expect("decrypt sum");
|
||||
|
||||
for (i, expected) in [5.0f32, 7.0, 9.0].iter().enumerate() {
|
||||
assert!(
|
||||
(sum[i] - expected).abs() < 0.001,
|
||||
"idx {}: {} vs {}",
|
||||
i,
|
||||
sum[i],
|
||||
expected,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
13
hivemind/src/crypto/mod.rs
Executable file
13
hivemind/src/crypto/mod.rs
Executable file
|
|
@ -0,0 +1,13 @@
|
|||
//! Cryptographic primitives for HiveMind.
|
||||
//!
|
||||
//! Contains gradient encryption:
|
||||
//! - `fhe` — AES-256-GCM wrapper (`GradientCryptoCtx`) used in V1.0
|
||||
//! - `fhe_real` — Real TFHE-based homomorphic encryption (feature-gated `fhe-real`)
|
||||
//!
|
||||
//! The `FheContext` alias in `fhe` will transparently upgrade to TFHE
|
||||
//! when the `fhe-real` feature is enabled.
|
||||
|
||||
pub mod fhe;
|
||||
|
||||
#[cfg(feature = "fhe-real")]
|
||||
pub mod fhe_real;
|
||||
145
hivemind/src/dht.rs
Executable file
145
hivemind/src/dht.rs
Executable file
|
|
@ -0,0 +1,145 @@
|
|||
/// Kademlia DHT operations for HiveMind.
|
||||
///
|
||||
/// Provides structured peer routing and distributed IoC storage using
|
||||
/// Kademlia's XOR distance metric. IoC records are stored in the DHT
|
||||
/// with TTL-based expiry to prevent stale threat data.
|
||||
use libp2p::{kad, Multiaddr, PeerId, Swarm};
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
use crate::bootstrap;
|
||||
use crate::transport::HiveMindBehaviour;
|
||||
|
||||
/// Store a threat indicator in the DHT.
|
||||
///
|
||||
/// The key is the serialized IoC identifier (e.g., JA4 hash or IP).
|
||||
/// Record is replicated to the `k` closest peers (Quorum::Majority).
|
||||
pub fn put_ioc_record(
|
||||
swarm: &mut Swarm<HiveMindBehaviour>,
|
||||
key_bytes: &[u8],
|
||||
value: Vec<u8>,
|
||||
local_peer_id: PeerId,
|
||||
) -> anyhow::Result<kad::QueryId> {
|
||||
let key = kad::RecordKey::new(&key_bytes);
|
||||
let record = kad::Record {
|
||||
key: key.clone(),
|
||||
value,
|
||||
publisher: Some(local_peer_id),
|
||||
expires: None, // Managed by MemoryStore TTL
|
||||
};
|
||||
|
||||
let query_id = swarm
|
||||
.behaviour_mut()
|
||||
.kademlia
|
||||
.put_record(record, kad::Quorum::Majority)
|
||||
.map_err(|e| anyhow::anyhow!("DHT put failed: {e:?}"))?;
|
||||
|
||||
debug!(?query_id, "DHT PUT initiated for IoC record");
|
||||
Ok(query_id)
|
||||
}
|
||||
|
||||
/// Look up a threat indicator in the DHT by key.
|
||||
pub fn get_ioc_record(
|
||||
swarm: &mut Swarm<HiveMindBehaviour>,
|
||||
key_bytes: &[u8],
|
||||
) -> kad::QueryId {
|
||||
let key = kad::RecordKey::new(&key_bytes);
|
||||
let query_id = swarm.behaviour_mut().kademlia.get_record(key);
|
||||
debug!(?query_id, "DHT GET initiated for IoC record");
|
||||
query_id
|
||||
}
|
||||
|
||||
/// Add a known peer address to the Kademlia routing table.
|
||||
///
|
||||
/// Rejects self-referencing entries (peer pointing to itself).
|
||||
pub fn add_peer(
|
||||
swarm: &mut Swarm<HiveMindBehaviour>,
|
||||
peer_id: &PeerId,
|
||||
addr: Multiaddr,
|
||||
local_peer_id: &PeerId,
|
||||
) {
|
||||
// SECURITY: Reject self-referencing entries
|
||||
if peer_id == local_peer_id {
|
||||
warn!(%peer_id, "Rejected self-referencing k-bucket entry");
|
||||
return;
|
||||
}
|
||||
|
||||
// SECURITY: Reject loopback/unspecified addresses
|
||||
if !bootstrap::is_routable_addr(&addr) {
|
||||
warn!(%peer_id, %addr, "Rejected non-routable address for k-bucket");
|
||||
return;
|
||||
}
|
||||
|
||||
swarm
|
||||
.behaviour_mut()
|
||||
.kademlia
|
||||
.add_address(peer_id, addr.clone());
|
||||
debug!(%peer_id, %addr, "Added peer to Kademlia routing table");
|
||||
}
|
||||
|
||||
/// Initiate a Kademlia bootstrap to populate routing table.
|
||||
pub fn bootstrap(swarm: &mut Swarm<HiveMindBehaviour>) -> anyhow::Result<kad::QueryId> {
|
||||
let query_id = swarm
|
||||
.behaviour_mut()
|
||||
.kademlia
|
||||
.bootstrap()
|
||||
.map_err(|e| anyhow::anyhow!("Kademlia bootstrap failed: {e:?}"))?;
|
||||
|
||||
info!(?query_id, "Kademlia bootstrap initiated");
|
||||
Ok(query_id)
|
||||
}
|
||||
|
||||
/// Handle a Kademlia event from the swarm event loop.
|
||||
pub fn handle_kad_event(event: kad::Event) {
|
||||
match event {
|
||||
kad::Event::OutboundQueryProgressed {
|
||||
id, result, step, ..
|
||||
} => match result {
|
||||
kad::QueryResult::GetRecord(Ok(kad::GetRecordOk::FoundRecord(
|
||||
kad::PeerRecord { record, .. },
|
||||
))) => {
|
||||
info!(
|
||||
?id,
|
||||
key_len = record.key.as_ref().len(),
|
||||
value_len = record.value.len(),
|
||||
"DHT record found"
|
||||
);
|
||||
}
|
||||
kad::QueryResult::GetRecord(Err(e)) => {
|
||||
warn!(?id, ?e, "DHT GET failed");
|
||||
}
|
||||
kad::QueryResult::PutRecord(Ok(kad::PutRecordOk { key })) => {
|
||||
info!(?id, key_len = key.as_ref().len(), "DHT PUT succeeded");
|
||||
}
|
||||
kad::QueryResult::PutRecord(Err(e)) => {
|
||||
warn!(?id, ?e, "DHT PUT failed");
|
||||
}
|
||||
kad::QueryResult::Bootstrap(Ok(kad::BootstrapOk {
|
||||
peer,
|
||||
num_remaining,
|
||||
})) => {
|
||||
info!(
|
||||
?id,
|
||||
%peer,
|
||||
num_remaining,
|
||||
step = step.count,
|
||||
"Kademlia bootstrap progress"
|
||||
);
|
||||
}
|
||||
kad::QueryResult::Bootstrap(Err(e)) => {
|
||||
warn!(?id, ?e, "Kademlia bootstrap failed");
|
||||
}
|
||||
_ => {
|
||||
debug!(?id, "Kademlia query progressed");
|
||||
}
|
||||
},
|
||||
kad::Event::RoutingUpdated {
|
||||
peer, addresses, ..
|
||||
} => {
|
||||
debug!(%peer, addr_count = addresses.len(), "Routing table updated");
|
||||
}
|
||||
kad::Event::RoutablePeer { peer, address } => {
|
||||
debug!(%peer, %address, "New routable peer discovered");
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
232
hivemind/src/gossip.rs
Executable file
232
hivemind/src/gossip.rs
Executable file
|
|
@ -0,0 +1,232 @@
|
|||
/// GossipSub operations for HiveMind.
|
||||
///
|
||||
/// Provides epidemic broadcast of IoC reports across the mesh.
|
||||
/// All messages are authenticated (Ed25519 signed) and deduplicated
|
||||
/// via content-hash message IDs.
|
||||
use common::hivemind::{self, IoC, ThreatReport};
|
||||
use libp2p::{gossipsub, PeerId, Swarm};
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
use crate::transport::HiveMindBehaviour;
|
||||
|
||||
/// Subscribe to all HiveMind GossipSub topics.
|
||||
pub fn subscribe_all(swarm: &mut Swarm<HiveMindBehaviour>) -> anyhow::Result<()> {
|
||||
let topics = [
|
||||
hivemind::topics::IOC_TOPIC,
|
||||
hivemind::topics::JA4_TOPIC,
|
||||
hivemind::topics::HEARTBEAT_TOPIC,
|
||||
hivemind::topics::A2A_VIOLATIONS_TOPIC,
|
||||
];
|
||||
|
||||
for topic_str in &topics {
|
||||
let topic = gossipsub::IdentTopic::new(*topic_str);
|
||||
swarm
|
||||
.behaviour_mut()
|
||||
.gossipsub
|
||||
.subscribe(&topic)
|
||||
.map_err(|e| anyhow::anyhow!("Failed to subscribe to {topic_str}: {e}"))?;
|
||||
info!(topic = topic_str, "Subscribed to GossipSub topic");
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Publish a ThreatReport to the IoC topic.
|
||||
pub fn publish_threat_report(
|
||||
swarm: &mut Swarm<HiveMindBehaviour>,
|
||||
report: &ThreatReport,
|
||||
) -> anyhow::Result<gossipsub::MessageId> {
|
||||
let topic = gossipsub::IdentTopic::new(hivemind::topics::IOC_TOPIC);
|
||||
let data = serde_json::to_vec(report)
|
||||
.map_err(|e| anyhow::anyhow!("Failed to serialize ThreatReport: {e}"))?;
|
||||
|
||||
// SECURITY: Enforce maximum message size
|
||||
if data.len() > hivemind::MAX_MESSAGE_SIZE {
|
||||
anyhow::bail!(
|
||||
"ThreatReport exceeds max message size ({} > {})",
|
||||
data.len(),
|
||||
hivemind::MAX_MESSAGE_SIZE
|
||||
);
|
||||
}
|
||||
|
||||
let msg_id = swarm
|
||||
.behaviour_mut()
|
||||
.gossipsub
|
||||
.publish(topic, data)
|
||||
.map_err(|e| anyhow::anyhow!("Failed to publish ThreatReport: {e}"))?;
|
||||
|
||||
debug!(?msg_id, "Published ThreatReport to IoC topic");
|
||||
Ok(msg_id)
|
||||
}
|
||||
|
||||
/// Publish a single IoC to the IoC topic as a lightweight message.
|
||||
pub fn publish_ioc(
|
||||
swarm: &mut Swarm<HiveMindBehaviour>,
|
||||
ioc: &IoC,
|
||||
) -> anyhow::Result<gossipsub::MessageId> {
|
||||
let topic = gossipsub::IdentTopic::new(hivemind::topics::IOC_TOPIC);
|
||||
let data = serde_json::to_vec(ioc)
|
||||
.map_err(|e| anyhow::anyhow!("Failed to serialize IoC: {e}"))?;
|
||||
|
||||
if data.len() > hivemind::MAX_MESSAGE_SIZE {
|
||||
anyhow::bail!("IoC message exceeds max size");
|
||||
}
|
||||
|
||||
let msg_id = swarm
|
||||
.behaviour_mut()
|
||||
.gossipsub
|
||||
.publish(topic, data)
|
||||
.map_err(|e| anyhow::anyhow!("Failed to publish IoC: {e}"))?;
|
||||
|
||||
debug!(?msg_id, "Published IoC");
|
||||
Ok(msg_id)
|
||||
}
|
||||
|
||||
/// Publish a JA4 fingerprint to the JA4 topic.
|
||||
pub fn publish_ja4(
|
||||
swarm: &mut Swarm<HiveMindBehaviour>,
|
||||
ja4_fingerprint: &str,
|
||||
src_ip: u32,
|
||||
) -> anyhow::Result<gossipsub::MessageId> {
|
||||
let topic = gossipsub::IdentTopic::new(hivemind::topics::JA4_TOPIC);
|
||||
|
||||
let payload = serde_json::json!({
|
||||
"ja4": ja4_fingerprint,
|
||||
"src_ip": src_ip,
|
||||
"timestamp": std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs(),
|
||||
});
|
||||
let data = serde_json::to_vec(&payload)
|
||||
.map_err(|e| anyhow::anyhow!("Failed to serialize JA4: {e}"))?;
|
||||
|
||||
let msg_id = swarm
|
||||
.behaviour_mut()
|
||||
.gossipsub
|
||||
.publish(topic, data)
|
||||
.map_err(|e| anyhow::anyhow!("Failed to publish JA4: {e}"))?;
|
||||
|
||||
debug!(?msg_id, ja4 = ja4_fingerprint, "Published JA4 fingerprint");
|
||||
Ok(msg_id)
|
||||
}
|
||||
|
||||
/// Publish a raw proof envelope to the A2A violations topic.
|
||||
///
|
||||
/// Called when proof data is ingested from the local blackwall-enterprise daemon (optional)
|
||||
/// via the TCP proof ingestion socket. The data is published verbatim —
|
||||
/// the hivemind node acts as a relay, not a parser.
|
||||
pub fn publish_proof_envelope(
|
||||
swarm: &mut Swarm<HiveMindBehaviour>,
|
||||
data: &[u8],
|
||||
) -> anyhow::Result<gossipsub::MessageId> {
|
||||
let topic = gossipsub::IdentTopic::new(hivemind::topics::A2A_VIOLATIONS_TOPIC);
|
||||
|
||||
// SECURITY: Enforce maximum message size
|
||||
if data.len() > hivemind::MAX_MESSAGE_SIZE {
|
||||
anyhow::bail!(
|
||||
"proof envelope exceeds max message size ({} > {})",
|
||||
data.len(),
|
||||
hivemind::MAX_MESSAGE_SIZE
|
||||
);
|
||||
}
|
||||
|
||||
let msg_id = swarm
|
||||
.behaviour_mut()
|
||||
.gossipsub
|
||||
.publish(topic, data.to_vec())
|
||||
.map_err(|e| anyhow::anyhow!("Failed to publish proof envelope: {e}"))?;
|
||||
|
||||
debug!(?msg_id, bytes = data.len(), "Published proof envelope to A2A violations topic");
|
||||
Ok(msg_id)
|
||||
}
|
||||
|
||||
/// Configure GossipSub topic scoring to penalize invalid messages.
|
||||
pub fn configure_topic_scoring(swarm: &mut Swarm<HiveMindBehaviour>) {
|
||||
let ioc_topic = gossipsub::IdentTopic::new(hivemind::topics::IOC_TOPIC);
|
||||
|
||||
let params = gossipsub::TopicScoreParams {
|
||||
topic_weight: 1.0,
|
||||
time_in_mesh_weight: 0.5,
|
||||
time_in_mesh_quantum: std::time::Duration::from_secs(1),
|
||||
first_message_deliveries_weight: 1.0,
|
||||
first_message_deliveries_cap: 20.0,
|
||||
// Heavy penalty for invalid/poisoned IoC messages
|
||||
invalid_message_deliveries_weight: -100.0,
|
||||
invalid_message_deliveries_decay: 0.1,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
swarm
|
||||
.behaviour_mut()
|
||||
.gossipsub
|
||||
.set_topic_params(ioc_topic, params)
|
||||
.ok(); // set_topic_params can fail if topic not subscribed yet
|
||||
}
|
||||
|
||||
/// Handle an incoming GossipSub message.
|
||||
///
|
||||
/// Returns the deserialized IoC if valid, or None if the message is
|
||||
/// malformed or fails basic validation.
|
||||
pub fn handle_gossip_message(
|
||||
propagation_source: PeerId,
|
||||
message: gossipsub::Message,
|
||||
) -> Option<IoC> {
|
||||
let topic = message.topic.as_str();
|
||||
|
||||
match topic {
|
||||
t if t == hivemind::topics::IOC_TOPIC => {
|
||||
match serde_json::from_slice::<IoC>(&message.data) {
|
||||
Ok(ioc) => {
|
||||
info!(
|
||||
%propagation_source,
|
||||
ioc_type = ioc.ioc_type,
|
||||
severity = ioc.severity,
|
||||
"Received IoC from peer"
|
||||
);
|
||||
// SECURITY: Single-peer IoC — track but don't trust yet
|
||||
// Cross-validation happens in consensus module (Phase 1)
|
||||
Some(ioc)
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(
|
||||
%propagation_source,
|
||||
error = %e,
|
||||
"Failed to deserialize IoC message — potential poisoning"
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
t if t == hivemind::topics::JA4_TOPIC => {
|
||||
debug!(
|
||||
%propagation_source,
|
||||
data_len = message.data.len(),
|
||||
"Received JA4 fingerprint from peer"
|
||||
);
|
||||
None // JA4 messages are informational, not IoC
|
||||
}
|
||||
t if t == hivemind::topics::HEARTBEAT_TOPIC => {
|
||||
debug!(%propagation_source, "Peer heartbeat received");
|
||||
None
|
||||
}
|
||||
t if t == hivemind::topics::A2A_VIOLATIONS_TOPIC => {
|
||||
info!(
|
||||
%propagation_source,
|
||||
bytes = message.data.len(),
|
||||
"Received A2A violation proof from peer"
|
||||
);
|
||||
// A2A proofs are informational — logged and counted by metrics.
|
||||
// Future: store for local policy enforcement or cross-validation.
|
||||
None
|
||||
}
|
||||
_ => {
|
||||
warn!(
|
||||
%propagation_source,
|
||||
topic,
|
||||
"Unknown GossipSub topic — ignoring"
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
141
hivemind/src/identity.rs
Executable file
141
hivemind/src/identity.rs
Executable file
|
|
@ -0,0 +1,141 @@
|
|||
//! Persistent node identity — load or generate Ed25519 keypair.
|
||||
//!
|
||||
//! On first launch the keypair is generated and saved to disk.
|
||||
//! Subsequent launches reuse the same identity so the PeerId is
|
||||
//! stable across restarts and reputation persists in the mesh.
|
||||
//!
|
||||
//! SECURITY: The key file is created with mode 0600 (owner-only).
|
||||
//! If the permissions are wrong at load time we refuse to start
|
||||
//! rather than risk using a compromised key.
|
||||
|
||||
use anyhow::Context;
|
||||
use libp2p::identity::Keypair;
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
use tracing::info;
|
||||
#[cfg(not(unix))]
|
||||
use tracing::warn;
|
||||
|
||||
/// Default directory for Blackwall identity and state.
|
||||
const DEFAULT_DATA_DIR: &str = ".blackwall";
|
||||
/// Filename for the Ed25519 secret key (PKCS#8 DER).
|
||||
const IDENTITY_FILENAME: &str = "identity.key";
|
||||
/// Expected Unix permissions (read/write owner only).
|
||||
#[cfg(unix)]
|
||||
const REQUIRED_MODE: u32 = 0o600;
|
||||
|
||||
/// Resolve the identity key path.
|
||||
///
|
||||
/// Priority:
|
||||
/// 1. Explicit path from config (`identity_key_path`)
|
||||
/// 2. `/etc/blackwall/identity.key` when running as root
|
||||
/// 3. `~/.blackwall/identity.key` otherwise
|
||||
pub fn resolve_key_path(explicit: Option<&str>) -> anyhow::Result<PathBuf> {
|
||||
if let Some(p) = explicit {
|
||||
return Ok(PathBuf::from(p));
|
||||
}
|
||||
|
||||
// Running as root → system-wide path
|
||||
#[cfg(unix)]
|
||||
if nix::unistd::geteuid().is_root() {
|
||||
return Ok(PathBuf::from("/etc/blackwall").join(IDENTITY_FILENAME));
|
||||
}
|
||||
|
||||
// Regular user → home dir
|
||||
let home = std::env::var("HOME")
|
||||
.or_else(|_| std::env::var("USERPROFILE"))
|
||||
.context("Cannot determine home directory (neither HOME nor USERPROFILE set)")?;
|
||||
Ok(PathBuf::from(home).join(DEFAULT_DATA_DIR).join(IDENTITY_FILENAME))
|
||||
}
|
||||
|
||||
/// Load an existing keypair or generate a new one and persist it.
|
||||
pub fn load_or_generate(key_path: &Path) -> anyhow::Result<Keypair> {
|
||||
if key_path.exists() {
|
||||
load_keypair(key_path)
|
||||
} else {
|
||||
generate_and_save(key_path)
|
||||
}
|
||||
}
|
||||
|
||||
/// Load keypair from disk, verifying file permissions first.
|
||||
fn load_keypair(path: &Path) -> anyhow::Result<Keypair> {
|
||||
verify_permissions(path)?;
|
||||
|
||||
let der = fs::read(path)
|
||||
.with_context(|| format!("Failed to read identity key: {}", path.display()))?;
|
||||
|
||||
let keypair = Keypair::from_protobuf_encoding(&der)
|
||||
.context("Failed to decode identity key (corrupt or wrong format?)")?;
|
||||
|
||||
info!(path = %path.display(), "Loaded persistent identity");
|
||||
Ok(keypair)
|
||||
}
|
||||
|
||||
/// Generate a fresh Ed25519 keypair, create parent dirs, save with 0600.
|
||||
fn generate_and_save(path: &Path) -> anyhow::Result<Keypair> {
|
||||
let keypair = Keypair::generate_ed25519();
|
||||
|
||||
// Create parent directory if missing
|
||||
if let Some(parent) = path.parent() {
|
||||
fs::create_dir_all(parent)
|
||||
.with_context(|| format!("Cannot create directory: {}", parent.display()))?;
|
||||
// SECURITY: restrict directory to owner-only on Unix
|
||||
#[cfg(unix)]
|
||||
set_permissions(parent, 0o700)?;
|
||||
}
|
||||
|
||||
// Serialize to protobuf (libp2p's canonical format)
|
||||
let encoded = keypair
|
||||
.to_protobuf_encoding()
|
||||
.context("Failed to encode keypair")?;
|
||||
|
||||
fs::write(path, &encoded)
|
||||
.with_context(|| format!("Failed to write identity key: {}", path.display()))?;
|
||||
|
||||
// SECURITY: set 0600 immediately after write
|
||||
#[cfg(unix)]
|
||||
set_permissions(path, REQUIRED_MODE)?;
|
||||
|
||||
info!(
|
||||
path = %path.display(),
|
||||
"Generated new persistent identity (saved to disk)"
|
||||
);
|
||||
Ok(keypair)
|
||||
}
|
||||
|
||||
/// Verify that the key file has strict permissions (Unix only).
|
||||
#[cfg(unix)]
|
||||
fn verify_permissions(path: &Path) -> anyhow::Result<()> {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
let meta = fs::metadata(path)
|
||||
.with_context(|| format!("Cannot stat identity key: {}", path.display()))?;
|
||||
let mode = meta.permissions().mode() & 0o777;
|
||||
if mode != REQUIRED_MODE {
|
||||
anyhow::bail!(
|
||||
"Identity key {} has insecure permissions {:04o} (expected {:04o}). \
|
||||
Fix with: chmod 600 {}",
|
||||
path.display(),
|
||||
mode,
|
||||
REQUIRED_MODE,
|
||||
path.display(),
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// No-op permission check on non-Unix platforms.
|
||||
#[cfg(not(unix))]
|
||||
fn verify_permissions(_path: &Path) -> anyhow::Result<()> {
|
||||
warn!("File permission check skipped (non-Unix platform)");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Set file/dir permissions (Unix only).
|
||||
#[cfg(unix)]
|
||||
fn set_permissions(path: &Path, mode: u32) -> anyhow::Result<()> {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
let perms = std::fs::Permissions::from_mode(mode);
|
||||
fs::set_permissions(path, perms)
|
||||
.with_context(|| format!("Cannot set permissions {:04o} on {}", mode, path.display()))?;
|
||||
Ok(())
|
||||
}
|
||||
24
hivemind/src/lib.rs
Executable file
24
hivemind/src/lib.rs
Executable file
|
|
@ -0,0 +1,24 @@
|
|||
//! HiveMind P2P Threat Intelligence Mesh.
|
||||
//!
|
||||
//! Library crate exposing the core P2P modules for the HiveMind daemon.
|
||||
//! Each module handles a specific aspect of the P2P networking stack:
|
||||
//!
|
||||
//! - `transport` — libp2p Swarm with Noise+QUIC, composite NetworkBehaviour
|
||||
//! - `dht` — Kademlia DHT for structured peer routing and IoC storage
|
||||
//! - `gossip` — GossipSub for epidemic IoC broadcast
|
||||
//! - `bootstrap` — Initial peer discovery (hardcoded nodes + mDNS)
|
||||
//! - `config` — TOML-based configuration
|
||||
|
||||
pub mod bootstrap;
|
||||
pub mod config;
|
||||
pub mod consensus;
|
||||
pub mod crypto;
|
||||
pub mod dht;
|
||||
pub mod gossip;
|
||||
pub mod identity;
|
||||
pub mod metrics_bridge;
|
||||
pub mod ml;
|
||||
pub mod reputation;
|
||||
pub mod sybil_guard;
|
||||
pub mod transport;
|
||||
pub mod zkp;
|
||||
871
hivemind/src/main.rs
Executable file
871
hivemind/src/main.rs
Executable file
|
|
@ -0,0 +1,871 @@
|
|||
/// HiveMind — P2P Threat Intelligence Mesh daemon.
|
||||
///
|
||||
/// Entry point for the HiveMind node. Builds the libp2p swarm,
|
||||
/// subscribes to GossipSub topics, connects to bootstrap nodes,
|
||||
/// and runs the event loop with consensus + reputation tracking.
|
||||
use anyhow::Context;
|
||||
use libp2p::{futures::StreamExt, swarm::SwarmEvent};
|
||||
use std::path::PathBuf;
|
||||
use tracing::{info, warn};
|
||||
|
||||
use hivemind::bootstrap;
|
||||
use hivemind::config::{self, HiveMindConfig, NodeMode};
|
||||
use hivemind::consensus::{ConsensusEngine, ConsensusResult};
|
||||
use hivemind::crypto::fhe::FheContext;
|
||||
use hivemind::dht;
|
||||
use hivemind::gossip;
|
||||
use hivemind::identity;
|
||||
use hivemind::metrics_bridge::{self, SharedP2pMetrics, P2pMetrics};
|
||||
use hivemind::ml::aggregator::FedAvgAggregator;
|
||||
use hivemind::ml::defense::{GradientDefense, GradientVerdict};
|
||||
use hivemind::ml::gradient_share;
|
||||
use hivemind::ml::local_model::LocalModel;
|
||||
use hivemind::reputation::ReputationStore;
|
||||
use hivemind::sybil_guard::SybilGuard;
|
||||
use hivemind::transport;
|
||||
use hivemind::zkp;
|
||||
|
||||
#[tokio::main(flavor = "current_thread")]
|
||||
async fn main() -> anyhow::Result<()> {
|
||||
// Initialize structured logging
|
||||
tracing_subscriber::fmt::init();
|
||||
|
||||
let config = load_or_default_config()?;
|
||||
|
||||
// --- Persistent identity ---
|
||||
let key_path = identity::resolve_key_path(config.identity_key_path.as_deref())
|
||||
.context("Cannot resolve identity key path")?;
|
||||
let keypair = identity::load_or_generate(&key_path)
|
||||
.context("Cannot load/generate identity keypair")?;
|
||||
|
||||
let mut swarm = transport::build_swarm(&config, keypair)
|
||||
.context("Failed to build HiveMind swarm")?;
|
||||
|
||||
let local_peer_id = *swarm.local_peer_id();
|
||||
info!(%local_peer_id, "HiveMind node starting");
|
||||
|
||||
// Start listening
|
||||
transport::start_listening(&mut swarm, &config)?;
|
||||
|
||||
// Subscribe to GossipSub topics
|
||||
gossip::subscribe_all(&mut swarm)?;
|
||||
|
||||
// Configure topic scoring (anti-poisoning)
|
||||
gossip::configure_topic_scoring(&mut swarm);
|
||||
|
||||
// Connect to bootstrap nodes
|
||||
let seed_peer_ids = bootstrap::connect_bootstrap_nodes(&mut swarm, &config, &local_peer_id)?;
|
||||
|
||||
// --- P2P metrics bridge (pushes live stats to hivemind-api) ---
|
||||
let p2p_metrics: SharedP2pMetrics = std::sync::Arc::new(P2pMetrics::default());
|
||||
|
||||
// Metrics push interval (5 seconds) — pushes P2P stats to hivemind-api
|
||||
let mut metrics_interval = tokio::time::interval(
|
||||
std::time::Duration::from_secs(5),
|
||||
);
|
||||
metrics_interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
|
||||
|
||||
info!(mode = ?config.mode, "HiveMind event loop starting");
|
||||
|
||||
match config.mode {
|
||||
NodeMode::Bootstrap => run_bootstrap_loop(&mut swarm, &p2p_metrics, metrics_interval).await,
|
||||
NodeMode::Full => run_full_loop(&mut swarm, &local_peer_id, &seed_peer_ids, &p2p_metrics, metrics_interval).await,
|
||||
}
|
||||
}
|
||||
|
||||
/// Lightweight bootstrap event loop — only Kademlia routing + GossipSub
|
||||
/// message forwarding + metrics push. No reputation, consensus, FL, or ZKP.
|
||||
///
|
||||
/// ARCH: Bootstrap nodes will also serve as Circuit Relay v2 destinations
|
||||
/// once NAT traversal is implemented (AutoNAT + Relay).
|
||||
async fn run_bootstrap_loop(
|
||||
swarm: &mut libp2p::Swarm<transport::HiveMindBehaviour>,
|
||||
p2p_metrics: &SharedP2pMetrics,
|
||||
mut metrics_interval: tokio::time::Interval,
|
||||
) -> anyhow::Result<()> {
|
||||
info!("Running in BOOTSTRAP mode — relay only (no DPI/AI/FL)");
|
||||
|
||||
loop {
|
||||
tokio::select! {
|
||||
event = swarm.select_next_some() => {
|
||||
handle_bootstrap_event(swarm, event, p2p_metrics);
|
||||
}
|
||||
_ = metrics_interval.tick() => {
|
||||
metrics_bridge::push_p2p_metrics(p2p_metrics).await;
|
||||
}
|
||||
_ = tokio::signal::ctrl_c() => {
|
||||
info!("Received SIGINT — shutting down bootstrap node");
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
info!("HiveMind bootstrap node shut down gracefully");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Full event loop — all modules active.
|
||||
async fn run_full_loop(
|
||||
swarm: &mut libp2p::Swarm<transport::HiveMindBehaviour>,
|
||||
local_peer_id: &libp2p::PeerId,
|
||||
seed_peer_ids: &[libp2p::PeerId],
|
||||
p2p_metrics: &SharedP2pMetrics,
|
||||
mut metrics_interval: tokio::time::Interval,
|
||||
) -> anyhow::Result<()> {
|
||||
// --- Phase 1: Anti-Poisoning modules ---
|
||||
let mut reputation = ReputationStore::new();
|
||||
let mut consensus = ConsensusEngine::new();
|
||||
let sybil_guard = SybilGuard::new();
|
||||
|
||||
// Register bootstrap nodes as seed peers with elevated stake so their
|
||||
// IoC reports are trusted immediately. Without this, INITIAL_STAKE < MIN_TRUSTED
|
||||
// means no peer can ever reach consensus.
|
||||
for peer_id in seed_peer_ids {
|
||||
let pubkey = peer_id_to_pubkey(peer_id);
|
||||
reputation.register_seed_peer(&pubkey);
|
||||
}
|
||||
// Also register self as seed peer — our own IoC submissions should count
|
||||
let local_pubkey_seed = peer_id_to_pubkey(local_peer_id);
|
||||
reputation.register_seed_peer(&local_pubkey_seed);
|
||||
|
||||
info!(
|
||||
seed_peers = seed_peer_ids.len() + 1,
|
||||
"Phase 1 security modules initialized (reputation, consensus, sybil_guard)"
|
||||
);
|
||||
|
||||
// --- Phase 2: Federated Learning modules ---
|
||||
let mut local_model = LocalModel::new(0.01);
|
||||
let fhe_ctx = FheContext::new();
|
||||
let mut aggregator = FedAvgAggregator::new();
|
||||
let mut gradient_defense = GradientDefense::new();
|
||||
|
||||
// Extract local node pubkey for gradient messages
|
||||
let local_pubkey = peer_id_to_pubkey(local_peer_id);
|
||||
|
||||
info!(
|
||||
model_params = local_model.param_count(),
|
||||
fhe_stub = fhe_ctx.is_stub(),
|
||||
"Phase 2 federated learning modules initialized"
|
||||
);
|
||||
|
||||
// Periodic eviction interval (5 minutes)
|
||||
let mut eviction_interval = tokio::time::interval(
|
||||
std::time::Duration::from_secs(common::hivemind::CONSENSUS_TIMEOUT_SECS),
|
||||
);
|
||||
eviction_interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
|
||||
|
||||
// Federated learning round interval (60 seconds)
|
||||
let mut fl_round_interval = tokio::time::interval(
|
||||
std::time::Duration::from_secs(common::hivemind::FL_ROUND_INTERVAL_SECS),
|
||||
);
|
||||
fl_round_interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
|
||||
|
||||
info!("Full-mode event loop starting");
|
||||
|
||||
// --- Proof ingestion socket (enterprise module → hivemind) ---
|
||||
let proof_addr = format!("127.0.0.1:{}", common::hivemind::PROOF_INGEST_PORT);
|
||||
let proof_listener = tokio::net::TcpListener::bind(&proof_addr)
|
||||
.await
|
||||
.context("failed to bind proof ingestion listener")?;
|
||||
info!(addr = %proof_addr, "proof ingestion listener ready");
|
||||
|
||||
// --- IoC injection socket (for testing/integration) ---
|
||||
let ioc_addr = format!("127.0.0.1:{}", common::hivemind::IOC_INJECT_PORT);
|
||||
let ioc_listener = tokio::net::TcpListener::bind(&ioc_addr)
|
||||
.await
|
||||
.context("failed to bind IoC injection listener")?;
|
||||
info!(addr = %ioc_addr, "IoC injection listener ready");
|
||||
|
||||
// Main event loop
|
||||
loop {
|
||||
tokio::select! {
|
||||
event = swarm.select_next_some() => {
|
||||
handle_swarm_event(
|
||||
swarm,
|
||||
event,
|
||||
local_peer_id,
|
||||
&mut reputation,
|
||||
&mut consensus,
|
||||
&fhe_ctx,
|
||||
&mut aggregator,
|
||||
&mut gradient_defense,
|
||||
&mut local_model,
|
||||
p2p_metrics,
|
||||
);
|
||||
}
|
||||
result = proof_listener.accept() => {
|
||||
if let Ok((stream, addr)) = result {
|
||||
tracing::debug!(%addr, "proof ingestion connection");
|
||||
ingest_proof_envelope(swarm, stream).await;
|
||||
}
|
||||
}
|
||||
result = ioc_listener.accept() => {
|
||||
if let Ok((stream, addr)) = result {
|
||||
tracing::debug!(%addr, "IoC injection connection");
|
||||
ingest_and_publish_ioc(
|
||||
swarm,
|
||||
stream,
|
||||
&local_pubkey,
|
||||
&mut reputation,
|
||||
&mut consensus,
|
||||
).await;
|
||||
}
|
||||
}
|
||||
_ = eviction_interval.tick() => {
|
||||
consensus.evict_expired();
|
||||
}
|
||||
_ = fl_round_interval.tick() => {
|
||||
// Federated Learning round: compute and broadcast gradients
|
||||
handle_fl_round(
|
||||
swarm,
|
||||
&mut local_model,
|
||||
&fhe_ctx,
|
||||
&mut aggregator,
|
||||
&local_pubkey,
|
||||
);
|
||||
}
|
||||
_ = metrics_interval.tick() => {
|
||||
metrics_bridge::push_p2p_metrics(p2p_metrics).await;
|
||||
}
|
||||
_ = tokio::signal::ctrl_c() => {
|
||||
info!("Received SIGINT — shutting down HiveMind");
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Log accepted IoCs before shutting down
|
||||
let final_accepted = consensus.drain_accepted();
|
||||
if !final_accepted.is_empty() {
|
||||
info!(
|
||||
count = final_accepted.len(),
|
||||
"Draining accepted IoCs at shutdown"
|
||||
);
|
||||
}
|
||||
// Suppress unused variable warnings until sybil_guard is wired
|
||||
// into the peer registration handshake protocol (Phase 2).
|
||||
let _ = &sybil_guard;
|
||||
|
||||
info!("HiveMind shut down gracefully");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Read a length-prefixed proof envelope from a TCP connection and
|
||||
/// publish it to GossipSub.
|
||||
///
|
||||
/// Wire format: `[4-byte big-endian length][JSON payload]`.
|
||||
async fn ingest_proof_envelope(
|
||||
swarm: &mut libp2p::Swarm<transport::HiveMindBehaviour>,
|
||||
mut stream: tokio::net::TcpStream,
|
||||
) {
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
// Read 4-byte length prefix
|
||||
let mut len_buf = [0u8; 4];
|
||||
if let Err(e) = stream.read_exact(&mut len_buf).await {
|
||||
warn!(error = %e, "proof ingestion: failed to read length prefix");
|
||||
return;
|
||||
}
|
||||
let len = u32::from_be_bytes(len_buf) as usize;
|
||||
if len == 0 || len > common::hivemind::MAX_MESSAGE_SIZE {
|
||||
warn!(len, "proof ingestion: invalid message length");
|
||||
return;
|
||||
}
|
||||
|
||||
// Read payload
|
||||
let mut buf = vec![0u8; len];
|
||||
if let Err(e) = stream.read_exact(&mut buf).await {
|
||||
warn!(error = %e, len, "proof ingestion: failed to read payload");
|
||||
return;
|
||||
}
|
||||
|
||||
// Publish to GossipSub
|
||||
match gossip::publish_proof_envelope(swarm, &buf) {
|
||||
Ok(msg_id) => {
|
||||
info!(?msg_id, bytes = len, "published ingested proof to mesh");
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(error = %e, "failed to publish ingested proof to GossipSub");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Read a length-prefixed IoC JSON from a TCP connection, publish it
|
||||
/// to GossipSub IOC topic, and submit to local consensus.
|
||||
///
|
||||
/// Wire format: `[4-byte big-endian length][JSON IoC payload]`.
|
||||
async fn ingest_and_publish_ioc(
|
||||
swarm: &mut libp2p::Swarm<transport::HiveMindBehaviour>,
|
||||
mut stream: tokio::net::TcpStream,
|
||||
local_pubkey: &[u8; 32],
|
||||
reputation: &mut ReputationStore,
|
||||
consensus: &mut ConsensusEngine,
|
||||
) {
|
||||
use common::hivemind::IoC;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
let mut len_buf = [0u8; 4];
|
||||
if let Err(e) = stream.read_exact(&mut len_buf).await {
|
||||
warn!(error = %e, "IoC inject: failed to read length prefix");
|
||||
return;
|
||||
}
|
||||
let len = u32::from_be_bytes(len_buf) as usize;
|
||||
if len == 0 || len > common::hivemind::MAX_MESSAGE_SIZE {
|
||||
warn!(len, "IoC inject: invalid message length");
|
||||
return;
|
||||
}
|
||||
|
||||
let mut buf = vec![0u8; len];
|
||||
if let Err(e) = stream.read_exact(&mut buf).await {
|
||||
warn!(error = %e, len, "IoC inject: failed to read payload");
|
||||
return;
|
||||
}
|
||||
|
||||
let ioc: IoC = match serde_json::from_slice(&buf) {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
warn!(error = %e, "IoC inject: invalid JSON");
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
// 1. Publish to GossipSub so other peers receive it
|
||||
match gossip::publish_ioc(swarm, &ioc) {
|
||||
Ok(msg_id) => {
|
||||
info!(?msg_id, ip = ioc.ip, "published injected IoC to mesh");
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(error = %e, "failed to publish injected IoC to GossipSub");
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Submit to local consensus with our own pubkey
|
||||
match consensus.submit_ioc(&ioc, local_pubkey) {
|
||||
ConsensusResult::Accepted(count) => {
|
||||
info!(count, ip = ioc.ip, "injected IoC reached consensus");
|
||||
reputation.record_accurate_report(local_pubkey);
|
||||
if ioc.ip != 0 {
|
||||
if let Err(e) = append_accepted_ioc(ioc.ip, ioc.severity, count as u8) {
|
||||
warn!("failed to persist accepted IoC: {}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
ConsensusResult::Pending(count) => {
|
||||
info!(count, ip = ioc.ip, "injected IoC pending cross-validation");
|
||||
}
|
||||
ConsensusResult::DuplicatePeer => {
|
||||
warn!(ip = ioc.ip, "injected IoC: duplicate peer submission");
|
||||
}
|
||||
ConsensusResult::Expired => {
|
||||
info!(ip = ioc.ip, "injected IoC: pending entry expired");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Load config from `hivemind.toml` in the current directory, or use defaults.
|
||||
fn load_or_default_config() -> anyhow::Result<HiveMindConfig> {
|
||||
let config_path = PathBuf::from("hivemind.toml");
|
||||
if config_path.exists() {
|
||||
let cfg = config::load_config(&config_path)
|
||||
.context("Failed to load hivemind.toml")?;
|
||||
info!(?config_path, "Configuration loaded");
|
||||
Ok(cfg)
|
||||
} else {
|
||||
info!("No hivemind.toml found — using default configuration");
|
||||
Ok(HiveMindConfig {
|
||||
mode: Default::default(),
|
||||
identity_key_path: None,
|
||||
network: Default::default(),
|
||||
bootstrap: Default::default(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Lightweight event handler for bootstrap mode.
|
||||
///
|
||||
/// Only processes Kademlia routing, GossipSub forwarding (no content
|
||||
/// inspection), mDNS discovery, Identify, and connection lifecycle.
|
||||
/// GossipSub messages are automatically forwarded by the protocol — we
|
||||
/// just need to update metrics and log connection events.
|
||||
fn handle_bootstrap_event(
|
||||
swarm: &mut libp2p::Swarm<transport::HiveMindBehaviour>,
|
||||
event: SwarmEvent<transport::HiveMindBehaviourEvent>,
|
||||
p2p_metrics: &SharedP2pMetrics,
|
||||
) {
|
||||
match event {
|
||||
SwarmEvent::Behaviour(transport::HiveMindBehaviourEvent::Kademlia(kad_event)) => {
|
||||
dht::handle_kad_event(kad_event);
|
||||
}
|
||||
SwarmEvent::Behaviour(transport::HiveMindBehaviourEvent::Gossipsub(
|
||||
libp2p::gossipsub::Event::Message { message_id, propagation_source, .. },
|
||||
)) => {
|
||||
// Bootstrap nodes only forward — no content inspection
|
||||
p2p_metrics.messages_total.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||
tracing::debug!(?message_id, %propagation_source, "Relayed GossipSub message");
|
||||
}
|
||||
SwarmEvent::Behaviour(transport::HiveMindBehaviourEvent::Gossipsub(
|
||||
libp2p::gossipsub::Event::Subscribed { peer_id, topic },
|
||||
)) => {
|
||||
info!(%peer_id, %topic, "Peer subscribed to topic");
|
||||
}
|
||||
SwarmEvent::Behaviour(transport::HiveMindBehaviourEvent::Gossipsub(
|
||||
libp2p::gossipsub::Event::Unsubscribed { peer_id, topic },
|
||||
)) => {
|
||||
info!(%peer_id, %topic, "Peer unsubscribed from topic");
|
||||
}
|
||||
SwarmEvent::Behaviour(transport::HiveMindBehaviourEvent::Gossipsub(_)) => {}
|
||||
SwarmEvent::Behaviour(transport::HiveMindBehaviourEvent::Mdns(
|
||||
libp2p::mdns::Event::Discovered(peers),
|
||||
)) => {
|
||||
let local = *swarm.local_peer_id();
|
||||
bootstrap::handle_mdns_discovered(swarm, peers, &local);
|
||||
}
|
||||
SwarmEvent::Behaviour(transport::HiveMindBehaviourEvent::Mdns(
|
||||
libp2p::mdns::Event::Expired(peers),
|
||||
)) => {
|
||||
bootstrap::handle_mdns_expired(swarm, peers);
|
||||
}
|
||||
SwarmEvent::Behaviour(transport::HiveMindBehaviourEvent::Identify(
|
||||
libp2p::identify::Event::Received { peer_id, info, .. },
|
||||
)) => {
|
||||
for addr in info.listen_addrs {
|
||||
if bootstrap::is_routable_addr(&addr) {
|
||||
swarm.behaviour_mut().kademlia.add_address(&peer_id, addr);
|
||||
}
|
||||
}
|
||||
}
|
||||
SwarmEvent::Behaviour(transport::HiveMindBehaviourEvent::Identify(_)) => {}
|
||||
SwarmEvent::NewListenAddr { address, .. } => {
|
||||
info!(%address, "New listen address");
|
||||
}
|
||||
SwarmEvent::ConnectionEstablished { peer_id, endpoint, .. } => {
|
||||
info!(%peer_id, ?endpoint, "Connection established");
|
||||
p2p_metrics.peer_count.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||
}
|
||||
SwarmEvent::ConnectionClosed { peer_id, cause, .. } => {
|
||||
info!(%peer_id, cause = ?cause, "Connection closed");
|
||||
let prev = p2p_metrics.peer_count.load(std::sync::atomic::Ordering::Relaxed);
|
||||
if prev > 0 {
|
||||
p2p_metrics.peer_count.fetch_sub(1, std::sync::atomic::Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
SwarmEvent::IncomingConnectionError { local_addr, error, .. } => {
|
||||
warn!(%local_addr, %error, "Incoming connection error");
|
||||
}
|
||||
SwarmEvent::OutgoingConnectionError { peer_id, error, .. } => {
|
||||
warn!(peer = ?peer_id, %error, "Outgoing connection error");
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
/// Dispatch swarm events to the appropriate handler module.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn handle_swarm_event(
|
||||
swarm: &mut libp2p::Swarm<transport::HiveMindBehaviour>,
|
||||
event: SwarmEvent<transport::HiveMindBehaviourEvent>,
|
||||
local_peer_id: &libp2p::PeerId,
|
||||
reputation: &mut ReputationStore,
|
||||
consensus: &mut ConsensusEngine,
|
||||
fhe_ctx: &FheContext,
|
||||
aggregator: &mut FedAvgAggregator,
|
||||
gradient_defense: &mut GradientDefense,
|
||||
local_model: &mut LocalModel,
|
||||
p2p_metrics: &SharedP2pMetrics,
|
||||
) {
|
||||
match event {
|
||||
// --- Kademlia events ---
|
||||
SwarmEvent::Behaviour(transport::HiveMindBehaviourEvent::Kademlia(kad_event)) => {
|
||||
dht::handle_kad_event(kad_event);
|
||||
}
|
||||
|
||||
// --- GossipSub events ---
|
||||
SwarmEvent::Behaviour(transport::HiveMindBehaviourEvent::Gossipsub(
|
||||
libp2p::gossipsub::Event::Message {
|
||||
propagation_source,
|
||||
message,
|
||||
message_id,
|
||||
..
|
||||
},
|
||||
)) => {
|
||||
info!(?message_id, %propagation_source, "GossipSub message received");
|
||||
p2p_metrics.messages_total.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||
|
||||
// Phase 2: Route gradient messages to FL handler
|
||||
if message.topic.as_str() == common::hivemind::topics::GRADIENT_TOPIC {
|
||||
if let Some(update) = gradient_share::handle_gradient_message(
|
||||
propagation_source,
|
||||
&message.data,
|
||||
) {
|
||||
handle_gradient_update(
|
||||
update,
|
||||
&propagation_source,
|
||||
fhe_ctx,
|
||||
aggregator,
|
||||
gradient_defense,
|
||||
local_model,
|
||||
reputation,
|
||||
);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if let Some(ioc) = gossip::handle_gossip_message(
|
||||
propagation_source,
|
||||
message.clone(),
|
||||
) {
|
||||
// Phase 1: Extract reporter pubkey from original publisher,
|
||||
// NOT propagation_source (which is the forwarding peer).
|
||||
// GossipSub MessageAuthenticity::Signed embeds the author.
|
||||
let author = message.source.unwrap_or(propagation_source);
|
||||
let reporter_pubkey = peer_id_to_pubkey(&author);
|
||||
|
||||
// Register peer if new (idempotent)
|
||||
reputation.register_peer(&reporter_pubkey);
|
||||
|
||||
// Verify ZKP proof if present
|
||||
if !ioc.zkp_proof.is_empty() {
|
||||
// Deserialize and verify the proof attached to the IoC
|
||||
if let Ok(proof) = serde_json::from_slice::<
|
||||
common::hivemind::ThreatProof,
|
||||
>(&ioc.zkp_proof) {
|
||||
let result = zkp::verifier::verify_threat(&proof, None);
|
||||
match result {
|
||||
zkp::verifier::VerifyResult::Valid
|
||||
| zkp::verifier::VerifyResult::ValidStub => {
|
||||
info!(%propagation_source, "ZKP proof verified");
|
||||
}
|
||||
other => {
|
||||
warn!(
|
||||
%propagation_source,
|
||||
result = ?other,
|
||||
"ZKP proof verification failed — untrusted IoC"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Submit to consensus — only trusted peers count
|
||||
if reputation.is_trusted(&reporter_pubkey) {
|
||||
match consensus.submit_ioc(&ioc, &reporter_pubkey) {
|
||||
ConsensusResult::Accepted(count) => {
|
||||
info!(
|
||||
count,
|
||||
ioc_type = ioc.ioc_type,
|
||||
"IoC reached consensus — adding to threat database"
|
||||
);
|
||||
reputation.record_accurate_report(&reporter_pubkey);
|
||||
// Persist accepted IoC IP for blackwall daemon ingestion
|
||||
if ioc.ip != 0 {
|
||||
if let Err(e) = append_accepted_ioc(
|
||||
ioc.ip,
|
||||
ioc.severity,
|
||||
count as u8,
|
||||
) {
|
||||
warn!("failed to persist accepted IoC: {}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
ConsensusResult::Pending(count) => {
|
||||
info!(
|
||||
count,
|
||||
threshold = common::hivemind::CROSS_VALIDATION_THRESHOLD,
|
||||
"IoC pending cross-validation"
|
||||
);
|
||||
}
|
||||
ConsensusResult::DuplicatePeer => {
|
||||
warn!(
|
||||
%propagation_source,
|
||||
"Duplicate IoC confirmation — ignoring"
|
||||
);
|
||||
}
|
||||
ConsensusResult::Expired => {
|
||||
info!("Pending IoC expired before consensus");
|
||||
}
|
||||
}
|
||||
} else {
|
||||
warn!(
|
||||
%propagation_source,
|
||||
stake = reputation.get_stake(&reporter_pubkey),
|
||||
"IoC from untrusted peer — ignoring"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
SwarmEvent::Behaviour(transport::HiveMindBehaviourEvent::Gossipsub(
|
||||
libp2p::gossipsub::Event::Subscribed { peer_id, topic },
|
||||
)) => {
|
||||
info!(%peer_id, %topic, "Peer subscribed to topic");
|
||||
}
|
||||
SwarmEvent::Behaviour(transport::HiveMindBehaviourEvent::Gossipsub(
|
||||
libp2p::gossipsub::Event::Unsubscribed { peer_id, topic },
|
||||
)) => {
|
||||
info!(%peer_id, %topic, "Peer unsubscribed from topic");
|
||||
}
|
||||
SwarmEvent::Behaviour(transport::HiveMindBehaviourEvent::Gossipsub(_)) => {}
|
||||
|
||||
// --- mDNS events ---
|
||||
SwarmEvent::Behaviour(transport::HiveMindBehaviourEvent::Mdns(
|
||||
libp2p::mdns::Event::Discovered(peers),
|
||||
)) => {
|
||||
bootstrap::handle_mdns_discovered(
|
||||
swarm,
|
||||
peers,
|
||||
local_peer_id,
|
||||
);
|
||||
}
|
||||
SwarmEvent::Behaviour(transport::HiveMindBehaviourEvent::Mdns(
|
||||
libp2p::mdns::Event::Expired(peers),
|
||||
)) => {
|
||||
bootstrap::handle_mdns_expired(swarm, peers);
|
||||
}
|
||||
|
||||
// --- Identify events ---
|
||||
SwarmEvent::Behaviour(transport::HiveMindBehaviourEvent::Identify(
|
||||
libp2p::identify::Event::Received { peer_id, info, .. },
|
||||
)) => {
|
||||
info!(
|
||||
%peer_id,
|
||||
protocol = %info.protocol_version,
|
||||
agent = %info.agent_version,
|
||||
"Identify: received peer info"
|
||||
);
|
||||
// Add identified addresses to Kademlia
|
||||
for addr in info.listen_addrs {
|
||||
if bootstrap::is_routable_addr(&addr) {
|
||||
swarm
|
||||
.behaviour_mut()
|
||||
.kademlia
|
||||
.add_address(&peer_id, addr);
|
||||
}
|
||||
}
|
||||
}
|
||||
SwarmEvent::Behaviour(transport::HiveMindBehaviourEvent::Identify(_)) => {}
|
||||
|
||||
// --- Connection lifecycle ---
|
||||
SwarmEvent::NewListenAddr { address, .. } => {
|
||||
info!(%address, "New listen address");
|
||||
}
|
||||
SwarmEvent::ConnectionEstablished {
|
||||
peer_id, endpoint, ..
|
||||
} => {
|
||||
info!(%peer_id, ?endpoint, "Connection established");
|
||||
p2p_metrics.peer_count.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||
}
|
||||
SwarmEvent::ConnectionClosed {
|
||||
peer_id, cause, ..
|
||||
} => {
|
||||
info!(
|
||||
%peer_id,
|
||||
cause = ?cause,
|
||||
"Connection closed"
|
||||
);
|
||||
// Saturating decrement
|
||||
let prev = p2p_metrics.peer_count.load(std::sync::atomic::Ordering::Relaxed);
|
||||
if prev > 0 {
|
||||
p2p_metrics.peer_count.fetch_sub(1, std::sync::atomic::Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
SwarmEvent::IncomingConnectionError {
|
||||
local_addr, error, ..
|
||||
} => {
|
||||
warn!(%local_addr, %error, "Incoming connection error");
|
||||
}
|
||||
SwarmEvent::OutgoingConnectionError {
|
||||
peer_id, error, ..
|
||||
} => {
|
||||
warn!(peer = ?peer_id, %error, "Outgoing connection error");
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
/// Extract a 32-byte public key representation from a PeerId.
|
||||
///
|
||||
/// PeerId is a multihash of the public key. We use the raw bytes
|
||||
/// truncated/padded to 32 bytes as a deterministic peer identifier
|
||||
/// for the reputation system.
|
||||
fn peer_id_to_pubkey(peer_id: &libp2p::PeerId) -> [u8; 32] {
|
||||
let bytes = peer_id.to_bytes();
|
||||
let mut pubkey = [0u8; 32];
|
||||
let len = bytes.len().min(32);
|
||||
pubkey[..len].copy_from_slice(&bytes[..len]);
|
||||
pubkey
|
||||
}
|
||||
|
||||
/// Handle an incoming gradient update from a peer.
|
||||
///
|
||||
/// Decrypts the FHE payload, runs defense checks, and submits to
|
||||
/// the aggregator if safe. When enough contributions arrive, triggers
|
||||
/// federated aggregation and model update.
|
||||
fn handle_gradient_update(
|
||||
update: common::hivemind::GradientUpdate,
|
||||
propagation_source: &libp2p::PeerId,
|
||||
fhe_ctx: &FheContext,
|
||||
aggregator: &mut FedAvgAggregator,
|
||||
gradient_defense: &mut GradientDefense,
|
||||
local_model: &mut LocalModel,
|
||||
reputation: &mut ReputationStore,
|
||||
) {
|
||||
// Decrypt gradients from FHE ciphertext
|
||||
let gradients = match fhe_ctx.decrypt_gradients(&update.encrypted_gradients) {
|
||||
Ok(g) => g,
|
||||
Err(e) => {
|
||||
warn!(
|
||||
%propagation_source,
|
||||
error = %e,
|
||||
"Failed to decrypt gradient payload"
|
||||
);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
// Run defense checks on decrypted gradients
|
||||
match gradient_defense.check(&gradients) {
|
||||
GradientVerdict::Safe => {}
|
||||
verdict => {
|
||||
warn!(
|
||||
%propagation_source,
|
||||
?verdict,
|
||||
"Gradient rejected by defense module"
|
||||
);
|
||||
// Slash reputation for bad gradient contributions
|
||||
let pubkey = peer_id_to_pubkey(propagation_source);
|
||||
reputation.record_false_report(&pubkey);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Submit to aggregator
|
||||
match aggregator.submit_gradients(
|
||||
&update.peer_pubkey,
|
||||
update.round_id,
|
||||
gradients,
|
||||
) {
|
||||
Ok(count) => {
|
||||
info!(
|
||||
count,
|
||||
round = update.round_id,
|
||||
"Gradient contribution accepted"
|
||||
);
|
||||
|
||||
// If enough peers contributed, aggregate and update model
|
||||
if aggregator.ready_to_aggregate() {
|
||||
match aggregator.aggregate() {
|
||||
Ok(agg_gradients) => {
|
||||
local_model.apply_gradients(&agg_gradients);
|
||||
info!(
|
||||
round = aggregator.current_round(),
|
||||
participants = count,
|
||||
"Federated model updated via FedAvg"
|
||||
);
|
||||
aggregator.advance_round();
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(error = %e, "Aggregation failed");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(
|
||||
%propagation_source,
|
||||
error = %e,
|
||||
"Gradient contribution rejected"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Periodic federated learning round handler.
|
||||
///
|
||||
/// Computes local gradients on a synthetic training sample, encrypts
|
||||
/// them via FHE, and broadcasts to the gradient topic.
|
||||
fn handle_fl_round(
|
||||
swarm: &mut libp2p::Swarm<transport::HiveMindBehaviour>,
|
||||
local_model: &mut LocalModel,
|
||||
fhe_ctx: &FheContext,
|
||||
aggregator: &mut FedAvgAggregator,
|
||||
local_pubkey: &[u8; 32],
|
||||
) {
|
||||
let round_id = aggregator.current_round();
|
||||
|
||||
// ARCH: In production, training data comes from local eBPF telemetry.
|
||||
// For now, use a synthetic "benign traffic" sample as a training signal.
|
||||
let synthetic_input = vec![0.5_f32; common::hivemind::FL_FEATURE_DIM];
|
||||
let synthetic_target = 0.0; // benign
|
||||
|
||||
// Forward and backward pass
|
||||
local_model.forward(&synthetic_input);
|
||||
let gradients = local_model.backward(synthetic_target);
|
||||
|
||||
// Encrypt gradients before transmission
|
||||
let encrypted = match fhe_ctx.encrypt_gradients(&gradients) {
|
||||
Ok(e) => e,
|
||||
Err(e) => {
|
||||
warn!(error = %e, "Failed to encrypt gradients — skipping FL round");
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
// Publish to the gradient topic
|
||||
match gradient_share::publish_gradients(swarm, local_pubkey, round_id, encrypted) {
|
||||
Ok(msg_id) => {
|
||||
info!(
|
||||
?msg_id,
|
||||
round_id,
|
||||
"Local gradients broadcasted for FL round"
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
// Expected to fail when no peers are connected — not an error
|
||||
warn!(error = %e, "Could not publish gradients (no peers?)");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Append an accepted IoC IP to the shared file for blackwall daemon ingestion.
|
||||
///
|
||||
/// Format: one JSON object per line with ip, severity, confidence, and
|
||||
/// block duration. The blackwall daemon polls this file, reads all lines,
|
||||
/// adds them to the BLOCKLIST with the prescribed TTL, and removes the file.
|
||||
/// Directory is created on first write if it doesn't exist.
|
||||
fn append_accepted_ioc(ip: u32, severity: u8, confirmations: u8) -> std::io::Result<()> {
|
||||
use std::io::Write;
|
||||
let dir = PathBuf::from("/run/blackwall");
|
||||
if !dir.exists() {
|
||||
info!(dir = %dir.display(), "creating /run/blackwall directory");
|
||||
std::fs::create_dir_all(&dir)?;
|
||||
}
|
||||
let path = dir.join("hivemind_accepted_iocs");
|
||||
|
||||
// Block duration scales with severity: high severity → longer block
|
||||
let duration_secs: u32 = match severity {
|
||||
0..=2 => 1800, // low: 30 min
|
||||
3..=5 => 3600, // medium: 1 hour
|
||||
6..=8 => 7200, // high: 2 hours
|
||||
_ => 14400, // critical: 4 hours
|
||||
};
|
||||
|
||||
info!(
|
||||
ip,
|
||||
severity,
|
||||
confirmations,
|
||||
duration_secs,
|
||||
path = %path.display(),
|
||||
"persisting accepted IoC to file"
|
||||
);
|
||||
|
||||
let mut file = std::fs::OpenOptions::new()
|
||||
.create(true)
|
||||
.append(true)
|
||||
.open(&path)?;
|
||||
writeln!(
|
||||
file,
|
||||
r#"{{"ip":{},"severity":{},"confirmations":{},"duration_secs":{}}}"#,
|
||||
ip, severity, confirmations, duration_secs,
|
||||
)?;
|
||||
info!("IoC persisted successfully");
|
||||
Ok(())
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Add a link
Reference in a new issue