mirror of
https://github.com/xzcrpw/blackwall.git
synced 2026-06-16 20:25:13 +02:00
v2.0.0: adaptive eBPF firewall with AI honeypot and P2P threat mesh
This commit is contained in:
commit
37c6bbf5a1
133 changed files with 28073 additions and 0 deletions
22
hivemind-api/Cargo.toml
Executable file
22
hivemind-api/Cargo.toml
Executable file
|
|
@ -0,0 +1,22 @@
|
|||
[package]
|
||||
name = "hivemind-api"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
description = "Enterprise Threat Feed API — REST/STIX/TAXII endpoint for HiveMind verified IoCs"
|
||||
|
||||
[dependencies]
|
||||
common = { path = "../common", default-features = false, features = ["user"] }
|
||||
tokio = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
tracing-subscriber = { workspace = true }
|
||||
anyhow = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
hyper = { workspace = true, features = ["server"] }
|
||||
hyper-util = { workspace = true }
|
||||
http-body-util = { workspace = true }
|
||||
ring = { workspace = true }
|
||||
|
||||
[[bin]]
|
||||
name = "hivemind-api"
|
||||
path = "src/main.rs"
|
||||
305
hivemind-api/src/feed.rs
Executable file
305
hivemind-api/src/feed.rs
Executable file
|
|
@ -0,0 +1,305 @@
|
|||
/// Feed query parameter parsing and response formatting.
|
||||
///
|
||||
/// Bridges the HTTP layer (server.rs) to the storage layer (store.rs)
|
||||
/// by parsing URL query parameters into `QueryParams` and formatting
|
||||
/// paginated feed results as JSON responses.
|
||||
use common::hivemind;
|
||||
|
||||
use crate::store::QueryParams;
|
||||
|
||||
/// Parse query parameters from a URI query string.
|
||||
///
|
||||
/// Supported parameters:
|
||||
/// - `since` — Unix timestamp filter (only IoCs verified after this time)
|
||||
/// - `severity` — Minimum severity level (0-4)
|
||||
/// - `type` — IoC type filter (0-4)
|
||||
/// - `limit` — Page size (capped by tier max)
|
||||
/// - `offset` — Pagination offset
|
||||
///
|
||||
/// Invalid parameter values are silently ignored (defaults used).
|
||||
pub fn parse_query_params(query: Option<&str>, max_page_size: usize) -> QueryParams {
|
||||
let mut params = QueryParams::new();
|
||||
params.limit = params.limit.min(max_page_size);
|
||||
|
||||
let query = match query {
|
||||
Some(q) => q,
|
||||
None => return params,
|
||||
};
|
||||
|
||||
for pair in query.split('&') {
|
||||
let mut parts = pair.splitn(2, '=');
|
||||
let key = match parts.next() {
|
||||
Some(k) => k,
|
||||
None => continue,
|
||||
};
|
||||
let value = match parts.next() {
|
||||
Some(v) => v,
|
||||
None => continue,
|
||||
};
|
||||
|
||||
match key {
|
||||
"since" => {
|
||||
if let Ok(ts) = value.parse::<u64>() {
|
||||
params.since = Some(ts);
|
||||
}
|
||||
}
|
||||
"severity" => {
|
||||
if let Ok(sev) = value.parse::<u8>() {
|
||||
if sev <= 4 {
|
||||
params.min_severity = Some(sev);
|
||||
}
|
||||
}
|
||||
}
|
||||
"type" => {
|
||||
if let Ok(t) = value.parse::<u8>() {
|
||||
if t <= 4 {
|
||||
params.ioc_type = Some(t);
|
||||
}
|
||||
}
|
||||
}
|
||||
"limit" => {
|
||||
if let Ok(l) = value.parse::<usize>() {
|
||||
params.limit = l.min(max_page_size).max(1);
|
||||
}
|
||||
}
|
||||
"offset" => {
|
||||
if let Ok(o) = value.parse::<usize>() {
|
||||
params.offset = o;
|
||||
}
|
||||
}
|
||||
_ => {} // Unknown params silently ignored
|
||||
}
|
||||
}
|
||||
|
||||
params
|
||||
}
|
||||
|
||||
/// Feed statistics for the /api/v1/stats endpoint.
|
||||
#[derive(Clone, Debug, serde::Serialize)]
|
||||
pub struct FeedStats {
|
||||
/// Total verified IoCs in the feed.
|
||||
pub total_iocs: usize,
|
||||
/// Breakdown by severity level.
|
||||
pub by_severity: SeverityBreakdown,
|
||||
/// Breakdown by IoC type.
|
||||
pub by_type: TypeBreakdown,
|
||||
}
|
||||
|
||||
/// Count of IoCs per severity level.
|
||||
#[derive(Clone, Debug, Default, serde::Serialize)]
|
||||
pub struct SeverityBreakdown {
|
||||
pub info: usize,
|
||||
pub low: usize,
|
||||
pub medium: usize,
|
||||
pub high: usize,
|
||||
pub critical: usize,
|
||||
}
|
||||
|
||||
/// Count of IoCs per type.
|
||||
#[derive(Clone, Debug, Default, serde::Serialize)]
|
||||
pub struct TypeBreakdown {
|
||||
pub malicious_ip: usize,
|
||||
pub ja4_fingerprint: usize,
|
||||
pub entropy_anomaly: usize,
|
||||
pub dns_tunnel: usize,
|
||||
pub behavioral_pattern: usize,
|
||||
}
|
||||
|
||||
/// Compute feed statistics from the store.
|
||||
pub fn compute_stats(store: &crate::store::ThreatFeedStore) -> FeedStats {
|
||||
let all = store.all();
|
||||
let mut by_severity = SeverityBreakdown::default();
|
||||
let mut by_type = TypeBreakdown::default();
|
||||
|
||||
for vioc in all {
|
||||
match hivemind::ThreatSeverity::from_u8(vioc.ioc.severity) {
|
||||
hivemind::ThreatSeverity::Info => by_severity.info += 1,
|
||||
hivemind::ThreatSeverity::Low => by_severity.low += 1,
|
||||
hivemind::ThreatSeverity::Medium => by_severity.medium += 1,
|
||||
hivemind::ThreatSeverity::High => by_severity.high += 1,
|
||||
hivemind::ThreatSeverity::Critical => by_severity.critical += 1,
|
||||
}
|
||||
|
||||
match hivemind::IoCType::from_u8(vioc.ioc.ioc_type) {
|
||||
hivemind::IoCType::MaliciousIp => by_type.malicious_ip += 1,
|
||||
hivemind::IoCType::Ja4Fingerprint => by_type.ja4_fingerprint += 1,
|
||||
hivemind::IoCType::EntropyAnomaly => by_type.entropy_anomaly += 1,
|
||||
hivemind::IoCType::DnsTunnel => by_type.dns_tunnel += 1,
|
||||
hivemind::IoCType::BehavioralPattern => by_type.behavioral_pattern += 1,
|
||||
}
|
||||
}
|
||||
|
||||
FeedStats {
|
||||
total_iocs: all.len(),
|
||||
by_severity,
|
||||
by_type,
|
||||
}
|
||||
}
|
||||
|
||||
/// Mesh stats compatible with the TUI dashboard's MeshStats struct.
|
||||
///
|
||||
/// Returns IoC counts mapped to the dashboard's expected fields.
|
||||
#[derive(Clone, Debug, serde::Serialize)]
|
||||
pub struct DashboardMeshStats {
|
||||
pub connected: bool,
|
||||
|
||||
// P2P Mesh
|
||||
pub peer_count: u64,
|
||||
pub dht_records: u64,
|
||||
pub gossip_topics: u64,
|
||||
pub messages_per_sec: f64,
|
||||
|
||||
// Threat Intel
|
||||
pub iocs_shared: u64,
|
||||
pub iocs_received: u64,
|
||||
pub avg_reputation: f64,
|
||||
|
||||
// Network Firewall (XDP/eBPF)
|
||||
pub packets_total: u64,
|
||||
pub packets_passed: u64,
|
||||
pub packets_dropped: u64,
|
||||
pub anomalies_sent: u64,
|
||||
|
||||
// A2A Firewall (separate from XDP)
|
||||
pub a2a_jwts_verified: u64,
|
||||
pub a2a_violations: u64,
|
||||
pub a2a_injections: u64,
|
||||
|
||||
// Cryptography
|
||||
pub zkp_proofs_generated: u64,
|
||||
pub zkp_proofs_verified: u64,
|
||||
pub fhe_encrypted: bool,
|
||||
}
|
||||
|
||||
/// Compute dashboard-compatible mesh stats from the store.
|
||||
pub fn compute_mesh_stats(store: &crate::store::ThreatFeedStore, counters: &crate::server::HivemindCounters) -> DashboardMeshStats {
|
||||
use std::sync::atomic::Ordering;
|
||||
let all = store.all();
|
||||
let total = all.len() as u64;
|
||||
|
||||
// eBPF/XDP counters
|
||||
let pkt_total = counters.packets_total.load(Ordering::Relaxed);
|
||||
let pkt_passed = counters.packets_passed.load(Ordering::Relaxed);
|
||||
let pkt_dropped = counters.packets_dropped.load(Ordering::Relaxed);
|
||||
let anomalies = counters.anomalies_sent.load(Ordering::Relaxed);
|
||||
|
||||
// P2P counters
|
||||
let peers = counters.peer_count.load(Ordering::Relaxed);
|
||||
let iocs_p2p = counters.iocs_shared_p2p.load(Ordering::Relaxed);
|
||||
let rep_x100 = counters.avg_reputation_x100.load(Ordering::Relaxed);
|
||||
let msgs_total = counters.messages_total.load(Ordering::Relaxed);
|
||||
|
||||
// A2A counters
|
||||
let a2a_jwts = counters.a2a_jwts_verified.load(Ordering::Relaxed);
|
||||
let a2a_viol = counters.a2a_violations.load(Ordering::Relaxed);
|
||||
let a2a_inj = counters.a2a_injections.load(Ordering::Relaxed);
|
||||
|
||||
DashboardMeshStats {
|
||||
connected: true,
|
||||
peer_count: peers,
|
||||
dht_records: total,
|
||||
gossip_topics: if total > 0 || peers > 0 { 1 } else { 0 },
|
||||
messages_per_sec: msgs_total as f64 / 60.0,
|
||||
iocs_shared: iocs_p2p,
|
||||
iocs_received: pkt_total,
|
||||
avg_reputation: rep_x100 as f64 / 100.0,
|
||||
packets_total: pkt_total,
|
||||
packets_passed: pkt_passed,
|
||||
packets_dropped: pkt_dropped,
|
||||
anomalies_sent: anomalies,
|
||||
a2a_jwts_verified: a2a_jwts,
|
||||
a2a_violations: a2a_viol,
|
||||
a2a_injections: a2a_inj,
|
||||
zkp_proofs_generated: 0,
|
||||
zkp_proofs_verified: 0,
|
||||
fhe_encrypted: false,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn parse_empty_query() {
|
||||
let params = parse_query_params(None, 1000);
|
||||
assert_eq!(params.limit, hivemind::API_DEFAULT_PAGE_SIZE);
|
||||
assert_eq!(params.offset, 0);
|
||||
assert!(params.since.is_none());
|
||||
assert!(params.min_severity.is_none());
|
||||
assert!(params.ioc_type.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_all_params() {
|
||||
let params = parse_query_params(
|
||||
Some("since=1700000000&severity=3&type=1&limit=50&offset=10"),
|
||||
1000,
|
||||
);
|
||||
assert_eq!(params.since, Some(1700000000));
|
||||
assert_eq!(params.min_severity, Some(3));
|
||||
assert_eq!(params.ioc_type, Some(1));
|
||||
assert_eq!(params.limit, 50);
|
||||
assert_eq!(params.offset, 10);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn limit_capped_by_tier() {
|
||||
let params = parse_query_params(Some("limit=5000"), 100);
|
||||
assert_eq!(params.limit, 100);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_params_ignored() {
|
||||
let params = parse_query_params(
|
||||
Some("since=notanumber&severity=99&limit=abc&unknown=foo"),
|
||||
1000,
|
||||
);
|
||||
assert!(params.since.is_none());
|
||||
assert!(params.min_severity.is_none()); // 99 > 4, ignored
|
||||
assert_eq!(params.limit, hivemind::API_DEFAULT_PAGE_SIZE);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compute_stats_populated() {
|
||||
use crate::store::ThreatFeedStore;
|
||||
use common::hivemind::IoC;
|
||||
|
||||
let mut store = ThreatFeedStore::new();
|
||||
store.insert(
|
||||
IoC {
|
||||
ioc_type: 0,
|
||||
severity: 3,
|
||||
ip: 1,
|
||||
ja4: None,
|
||||
entropy_score: None,
|
||||
description: "test".to_string(),
|
||||
first_seen: 1000,
|
||||
confirmations: 3,
|
||||
zkp_proof: Vec::new(),
|
||||
},
|
||||
2000,
|
||||
);
|
||||
store.insert(
|
||||
IoC {
|
||||
ioc_type: 1,
|
||||
severity: 4,
|
||||
ip: 2,
|
||||
ja4: None,
|
||||
entropy_score: None,
|
||||
description: "test2".to_string(),
|
||||
first_seen: 1000,
|
||||
confirmations: 3,
|
||||
zkp_proof: Vec::new(),
|
||||
},
|
||||
3000,
|
||||
);
|
||||
|
||||
let stats = compute_stats(&store);
|
||||
assert_eq!(stats.total_iocs, 2);
|
||||
assert_eq!(stats.by_severity.high, 1);
|
||||
assert_eq!(stats.by_severity.critical, 1);
|
||||
assert_eq!(stats.by_type.malicious_ip, 1);
|
||||
assert_eq!(stats.by_type.ja4_fingerprint, 1);
|
||||
}
|
||||
}
|
||||
187
hivemind-api/src/integrations/cef.rs
Executable file
187
hivemind-api/src/integrations/cef.rs
Executable file
|
|
@ -0,0 +1,187 @@
|
|||
/// ArcSight Common Event Format (CEF) exporter.
|
||||
///
|
||||
/// Converts verified IoCs to CEF format for ingestion by ArcSight,
|
||||
/// Sentinel, and other SIEM platforms that support CEF.
|
||||
///
|
||||
/// Format: `CEF:0|Vendor|Product|Version|SignatureID|Name|Severity|Extensions`
|
||||
///
|
||||
/// Reference: <https://www.microfocus.com/documentation/arcsight/arcsight-smartconnectors/pdfdoc/cef-implementation-standard/cef-implementation-standard.pdf>
|
||||
use common::hivemind::{self, IoCType, ThreatSeverity};
|
||||
|
||||
use crate::store::{ip_to_string, unix_to_iso8601, VerifiedIoC};
|
||||
|
||||
/// Convert a verified IoC to a CEF format string.
|
||||
///
|
||||
/// The returned string is a single CEF event line.
|
||||
pub fn ioc_to_cef(vioc: &VerifiedIoC) -> String {
|
||||
let ioc = &vioc.ioc;
|
||||
let ioc_type = IoCType::from_u8(ioc.ioc_type);
|
||||
let severity = ThreatSeverity::from_u8(ioc.severity);
|
||||
|
||||
let sig_id = cef_signature_id(ioc_type);
|
||||
let name = escape_cef_header(&cef_event_name(ioc_type, ioc));
|
||||
let sev = cef_severity(severity);
|
||||
|
||||
// CEF header (pipe-delimited, 7 fields after CEF:0)
|
||||
let header = format!(
|
||||
"CEF:0|{}|{}|{}|{sig_id}|{name}|{sev}",
|
||||
escape_cef_header(hivemind::SIEM_VENDOR),
|
||||
escape_cef_header(hivemind::SIEM_PRODUCT),
|
||||
escape_cef_header(hivemind::SIEM_VERSION),
|
||||
);
|
||||
|
||||
// CEF extensions (key=value space-delimited)
|
||||
let src_ip = ip_to_string(ioc.ip);
|
||||
let timestamp = unix_to_iso8601(vioc.verified_at);
|
||||
|
||||
let mut ext = format!(
|
||||
"src={src_ip} rt={timestamp} cat={} msg={} cs1Label=stix_id cs1={} \
|
||||
cn1Label=confirmations cn1={}",
|
||||
escape_cef_value(cef_category(ioc_type)),
|
||||
escape_cef_value(&ioc.description),
|
||||
escape_cef_value(&vioc.stix_id),
|
||||
ioc.confirmations,
|
||||
);
|
||||
|
||||
if let Some(ref ja4) = ioc.ja4 {
|
||||
ext.push_str(&format!(
|
||||
" cs2Label=ja4 cs2={}",
|
||||
escape_cef_value(ja4)
|
||||
));
|
||||
}
|
||||
|
||||
if let Some(entropy) = ioc.entropy_score {
|
||||
ext.push_str(&format!(" cn2Label=entropy_score cn2={entropy}"));
|
||||
}
|
||||
|
||||
format!("{header}|{ext}")
|
||||
}
|
||||
|
||||
/// Convert a batch of verified IoCs to newline-delimited CEF.
|
||||
pub fn batch_to_cef(iocs: &[VerifiedIoC]) -> String {
|
||||
iocs.iter()
|
||||
.map(ioc_to_cef)
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n")
|
||||
}
|
||||
|
||||
/// Map IoC type to CEF signature ID.
|
||||
fn cef_signature_id(t: IoCType) -> u16 {
|
||||
match t {
|
||||
IoCType::MaliciousIp => 1001,
|
||||
IoCType::Ja4Fingerprint => 1002,
|
||||
IoCType::EntropyAnomaly => 1003,
|
||||
IoCType::DnsTunnel => 1004,
|
||||
IoCType::BehavioralPattern => 1005,
|
||||
}
|
||||
}
|
||||
|
||||
/// Build human-readable CEF event name.
|
||||
fn cef_event_name(t: IoCType, ioc: &common::hivemind::IoC) -> String {
|
||||
match t {
|
||||
IoCType::MaliciousIp => format!("Malicious IP {}", ip_to_string(ioc.ip)),
|
||||
IoCType::Ja4Fingerprint => "Malicious TLS Fingerprint".to_string(),
|
||||
IoCType::EntropyAnomaly => "High Entropy Anomaly".to_string(),
|
||||
IoCType::DnsTunnel => "DNS Tunneling Detected".to_string(),
|
||||
IoCType::BehavioralPattern => "Behavioral Anomaly".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Map threat severity to CEF severity (0-10).
|
||||
fn cef_severity(s: ThreatSeverity) -> u8 {
|
||||
match s {
|
||||
ThreatSeverity::Info => 1,
|
||||
ThreatSeverity::Low => 3,
|
||||
ThreatSeverity::Medium => 5,
|
||||
ThreatSeverity::High => 8,
|
||||
ThreatSeverity::Critical => 10,
|
||||
}
|
||||
}
|
||||
|
||||
/// Map IoC type to CEF category string.
|
||||
fn cef_category(t: IoCType) -> &'static str {
|
||||
match t {
|
||||
IoCType::MaliciousIp => "Threat/MaliciousIP",
|
||||
IoCType::Ja4Fingerprint => "Threat/TLSFingerprint",
|
||||
IoCType::EntropyAnomaly => "Anomaly/Entropy",
|
||||
IoCType::DnsTunnel => "Threat/DNSTunnel",
|
||||
IoCType::BehavioralPattern => "Anomaly/Behavioral",
|
||||
}
|
||||
}
|
||||
|
||||
/// Escape pipe characters in CEF header fields.
|
||||
///
|
||||
/// CEF uses `|` as the header delimiter — pipes must be escaped as `\|`.
|
||||
fn escape_cef_header(s: &str) -> String {
|
||||
s.replace('\\', "\\\\").replace('|', "\\|")
|
||||
}
|
||||
|
||||
/// Escape special characters in CEF extension values.
|
||||
///
|
||||
/// Backslash, equals, and newlines must be escaped in extension values.
|
||||
fn escape_cef_value(s: &str) -> String {
|
||||
s.replace('\\', "\\\\")
|
||||
.replace('=', "\\=")
|
||||
.replace('\n', "\\n")
|
||||
.replace('\r', "\\r")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use common::hivemind::IoC;
|
||||
|
||||
fn sample_vioc() -> VerifiedIoC {
|
||||
VerifiedIoC {
|
||||
ioc: IoC {
|
||||
ioc_type: 0,
|
||||
severity: 4,
|
||||
ip: 0x0A000001,
|
||||
ja4: Some("t13d".to_string()),
|
||||
entropy_score: Some(8000),
|
||||
description: "Critical threat".to_string(),
|
||||
first_seen: 1700000000,
|
||||
confirmations: 5,
|
||||
zkp_proof: Vec::new(),
|
||||
},
|
||||
verified_at: 1700001000,
|
||||
stix_id: "indicator--test".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cef_format_structure() {
|
||||
let vioc = sample_vioc();
|
||||
let cef = ioc_to_cef(&vioc);
|
||||
|
||||
// CEF header has 8 pipe-delimited fields
|
||||
let parts: Vec<&str> = cef.splitn(8, '|').collect();
|
||||
assert_eq!(parts.len(), 8);
|
||||
assert_eq!(parts[0], "CEF:0");
|
||||
assert_eq!(parts[1], "Blackwall");
|
||||
assert_eq!(parts[2], "HiveMind");
|
||||
assert_eq!(parts[3], "1.0");
|
||||
assert_eq!(parts[4], "1001"); // MaliciousIp signature ID
|
||||
assert!(parts[5].contains("10.0.0.1"));
|
||||
assert_eq!(parts[6], "10"); // Critical severity
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cef_escapes_pipes() {
|
||||
assert_eq!(escape_cef_header("test|pipe"), "test\\|pipe");
|
||||
assert_eq!(escape_cef_header("back\\slash"), "back\\\\slash");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cef_escapes_extension_values() {
|
||||
assert_eq!(escape_cef_value("key=value"), "key\\=value");
|
||||
assert_eq!(escape_cef_value("line\nnew"), "line\\nnew");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cef_severity_mapping() {
|
||||
assert_eq!(cef_severity(ThreatSeverity::Info), 1);
|
||||
assert_eq!(cef_severity(ThreatSeverity::High), 8);
|
||||
assert_eq!(cef_severity(ThreatSeverity::Critical), 10);
|
||||
}
|
||||
}
|
||||
11
hivemind-api/src/integrations/mod.rs
Executable file
11
hivemind-api/src/integrations/mod.rs
Executable file
|
|
@ -0,0 +1,11 @@
|
|||
//! SIEM/SOAR integration format exporters.
|
||||
//!
|
||||
//! Converts verified IoCs to industry-standard SIEM ingestion formats:
|
||||
//!
|
||||
//! - `splunk` — Splunk HTTP Event Collector (HEC) JSON format
|
||||
//! - `qradar` — IBM QRadar LEEF (Log Event Extended Format)
|
||||
//! - `cef` — ArcSight Common Event Format (CEF)
|
||||
|
||||
pub mod cef;
|
||||
pub mod qradar;
|
||||
pub mod splunk;
|
||||
142
hivemind-api/src/integrations/qradar.rs
Executable file
142
hivemind-api/src/integrations/qradar.rs
Executable file
|
|
@ -0,0 +1,142 @@
|
|||
/// IBM QRadar LEEF (Log Event Extended Format) exporter.
|
||||
///
|
||||
/// Converts verified IoCs to LEEF 2.0 format for ingestion by
|
||||
/// IBM QRadar SIEM via log source or Syslog.
|
||||
///
|
||||
/// LEEF format: `LEEF:2.0|Vendor|Product|Version|EventID\tkey=value\tkey=value`
|
||||
///
|
||||
/// Reference: <https://www.ibm.com/docs/en/dsm?topic=leef-overview>
|
||||
use common::hivemind::{self, IoCType, ThreatSeverity};
|
||||
|
||||
use crate::store::{ip_to_string, unix_to_iso8601, VerifiedIoC};
|
||||
|
||||
/// Convert a verified IoC to LEEF 2.0 format string.
|
||||
///
|
||||
/// The returned string is a single LEEF event line suitable for
|
||||
/// Syslog forwarding or file-based ingestion.
|
||||
pub fn ioc_to_leef(vioc: &VerifiedIoC) -> String {
|
||||
let ioc = &vioc.ioc;
|
||||
let ioc_type = IoCType::from_u8(ioc.ioc_type);
|
||||
let severity = ThreatSeverity::from_u8(ioc.severity);
|
||||
|
||||
let event_id = leef_event_id(ioc_type);
|
||||
let sev = leef_severity(severity);
|
||||
|
||||
// LEEF header
|
||||
let header = format!(
|
||||
"LEEF:2.0|{}|{}|{}|{}",
|
||||
hivemind::SIEM_VENDOR,
|
||||
hivemind::SIEM_PRODUCT,
|
||||
hivemind::SIEM_VERSION,
|
||||
event_id,
|
||||
);
|
||||
|
||||
// LEEF attributes (tab-delimited)
|
||||
let src_ip = ip_to_string(ioc.ip);
|
||||
let timestamp = unix_to_iso8601(vioc.verified_at);
|
||||
let desc = escape_leef_value(&ioc.description);
|
||||
|
||||
let mut attrs = format!(
|
||||
"sev={sev}\tsrc={src_ip}\tdevTime={timestamp}\tcat={event_id}\t\
|
||||
msg={desc}\tconfirmations={}\tstix_id={}",
|
||||
ioc.confirmations,
|
||||
escape_leef_value(&vioc.stix_id),
|
||||
);
|
||||
|
||||
if let Some(ref ja4) = ioc.ja4 {
|
||||
attrs.push_str(&format!("\tja4={}", escape_leef_value(ja4)));
|
||||
}
|
||||
|
||||
if let Some(entropy) = ioc.entropy_score {
|
||||
attrs.push_str(&format!("\tentropy_score={entropy}"));
|
||||
}
|
||||
|
||||
format!("{header}\t{attrs}")
|
||||
}
|
||||
|
||||
/// Convert a batch of verified IoCs to newline-delimited LEEF.
|
||||
pub fn batch_to_leef(iocs: &[VerifiedIoC]) -> String {
|
||||
iocs.iter()
|
||||
.map(ioc_to_leef)
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n")
|
||||
}
|
||||
|
||||
/// Map IoC type to LEEF event ID.
|
||||
fn leef_event_id(t: IoCType) -> &'static str {
|
||||
match t {
|
||||
IoCType::MaliciousIp => "MaliciousIP",
|
||||
IoCType::Ja4Fingerprint => "JA4Fingerprint",
|
||||
IoCType::EntropyAnomaly => "EntropyAnomaly",
|
||||
IoCType::DnsTunnel => "DNSTunnel",
|
||||
IoCType::BehavioralPattern => "BehavioralPattern",
|
||||
}
|
||||
}
|
||||
|
||||
/// Map threat severity to LEEF numeric severity (1-10).
|
||||
fn leef_severity(s: ThreatSeverity) -> u8 {
|
||||
match s {
|
||||
ThreatSeverity::Info => 1,
|
||||
ThreatSeverity::Low => 3,
|
||||
ThreatSeverity::Medium => 5,
|
||||
ThreatSeverity::High => 7,
|
||||
ThreatSeverity::Critical => 10,
|
||||
}
|
||||
}
|
||||
|
||||
/// Escape special characters in LEEF attribute values.
|
||||
///
|
||||
/// LEEF uses tab as delimiter — tabs and newlines must be escaped.
|
||||
fn escape_leef_value(s: &str) -> String {
|
||||
s.replace('\t', "\\t")
|
||||
.replace('\n', "\\n")
|
||||
.replace('\r', "\\r")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use common::hivemind::IoC;
|
||||
|
||||
fn sample_vioc() -> VerifiedIoC {
|
||||
VerifiedIoC {
|
||||
ioc: IoC {
|
||||
ioc_type: 0,
|
||||
severity: 3,
|
||||
ip: 0xC0A80001,
|
||||
ja4: Some("t13d1516h2_abc".to_string()),
|
||||
entropy_score: Some(7500),
|
||||
description: "Malicious IP detected".to_string(),
|
||||
first_seen: 1700000000,
|
||||
confirmations: 3,
|
||||
zkp_proof: Vec::new(),
|
||||
},
|
||||
verified_at: 1700001000,
|
||||
stix_id: "indicator--aabb".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn leef_header_format() {
|
||||
let vioc = sample_vioc();
|
||||
let leef = ioc_to_leef(&vioc);
|
||||
|
||||
assert!(leef.starts_with("LEEF:2.0|Blackwall|HiveMind|1.0|MaliciousIP"));
|
||||
assert!(leef.contains("sev=7"));
|
||||
assert!(leef.contains("src=192.168.0.1"));
|
||||
assert!(leef.contains("ja4=t13d1516h2_abc"));
|
||||
assert!(leef.contains("entropy_score=7500"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn leef_escapes_special_chars() {
|
||||
let escaped = escape_leef_value("test\ttab\nnewline");
|
||||
assert_eq!(escaped, "test\\ttab\\nnewline");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn leef_severity_mapping() {
|
||||
assert_eq!(leef_severity(ThreatSeverity::Info), 1);
|
||||
assert_eq!(leef_severity(ThreatSeverity::Critical), 10);
|
||||
}
|
||||
}
|
||||
159
hivemind-api/src/integrations/splunk.rs
Executable file
159
hivemind-api/src/integrations/splunk.rs
Executable file
|
|
@ -0,0 +1,159 @@
|
|||
/// Splunk HTTP Event Collector (HEC) format exporter.
|
||||
///
|
||||
/// Converts verified IoCs to Splunk HEC JSON format suitable for
|
||||
/// direct ingestion via the Splunk HEC endpoint.
|
||||
///
|
||||
/// Format reference: <https://docs.splunk.com/Documentation/Splunk/latest/Data/FormateventsforHTTPEventCollector>
|
||||
use common::hivemind::{self, IoCType, ThreatSeverity};
|
||||
use serde::Serialize;
|
||||
|
||||
use crate::store::{ip_to_string, VerifiedIoC};
|
||||
|
||||
/// Splunk HEC event wrapper.
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
pub struct SplunkEvent {
|
||||
/// Unix timestamp of the event.
|
||||
pub time: u64,
|
||||
/// Splunk source identifier.
|
||||
pub source: &'static str,
|
||||
/// Splunk sourcetype for indexing.
|
||||
pub sourcetype: &'static str,
|
||||
/// Target Splunk index.
|
||||
pub index: &'static str,
|
||||
/// Event payload.
|
||||
pub event: SplunkEventData,
|
||||
}
|
||||
|
||||
/// Inner event data for Splunk HEC.
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
pub struct SplunkEventData {
|
||||
/// IoC type as human-readable string.
|
||||
pub ioc_type: &'static str,
|
||||
/// Severity as human-readable string.
|
||||
pub severity: &'static str,
|
||||
/// Numeric severity (0-4).
|
||||
pub severity_id: u8,
|
||||
/// Source IP in dotted notation (if applicable).
|
||||
pub src_ip: String,
|
||||
/// JA4 fingerprint (if applicable).
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub ja4: Option<String>,
|
||||
/// Byte diversity score (if applicable).
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub entropy_score: Option<u32>,
|
||||
/// Human-readable description.
|
||||
pub description: String,
|
||||
/// Number of independent confirmations.
|
||||
pub confirmations: u32,
|
||||
/// Unix timestamp when first observed.
|
||||
pub first_seen: u64,
|
||||
/// Unix timestamp when consensus was reached.
|
||||
pub verified_at: u64,
|
||||
/// STIX identifier for cross-referencing.
|
||||
pub stix_id: String,
|
||||
}
|
||||
|
||||
/// Convert a verified IoC to a Splunk HEC event.
|
||||
pub fn ioc_to_splunk(vioc: &VerifiedIoC) -> SplunkEvent {
|
||||
let ioc = &vioc.ioc;
|
||||
let ioc_type = IoCType::from_u8(ioc.ioc_type);
|
||||
let severity = ThreatSeverity::from_u8(ioc.severity);
|
||||
|
||||
SplunkEvent {
|
||||
time: vioc.verified_at,
|
||||
source: "hivemind",
|
||||
sourcetype: hivemind::SPLUNK_SOURCETYPE,
|
||||
index: "threat_intel",
|
||||
event: SplunkEventData {
|
||||
ioc_type: ioc_type_label(ioc_type),
|
||||
severity: severity_label(severity),
|
||||
severity_id: ioc.severity,
|
||||
src_ip: ip_to_string(ioc.ip),
|
||||
ja4: ioc.ja4.clone(),
|
||||
entropy_score: ioc.entropy_score,
|
||||
description: ioc.description.clone(),
|
||||
confirmations: ioc.confirmations,
|
||||
first_seen: ioc.first_seen,
|
||||
verified_at: vioc.verified_at,
|
||||
stix_id: vioc.stix_id.clone(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert a batch of verified IoCs to Splunk HEC events.
|
||||
pub fn batch_to_splunk(iocs: &[VerifiedIoC]) -> Vec<SplunkEvent> {
|
||||
iocs.iter().map(ioc_to_splunk).collect()
|
||||
}
|
||||
|
||||
/// Human-readable IoC type label.
|
||||
fn ioc_type_label(t: IoCType) -> &'static str {
|
||||
match t {
|
||||
IoCType::MaliciousIp => "malicious_ip",
|
||||
IoCType::Ja4Fingerprint => "ja4_fingerprint",
|
||||
IoCType::EntropyAnomaly => "entropy_anomaly",
|
||||
IoCType::DnsTunnel => "dns_tunnel",
|
||||
IoCType::BehavioralPattern => "behavioral_pattern",
|
||||
}
|
||||
}
|
||||
|
||||
/// Human-readable severity label.
|
||||
fn severity_label(s: ThreatSeverity) -> &'static str {
|
||||
match s {
|
||||
ThreatSeverity::Info => "info",
|
||||
ThreatSeverity::Low => "low",
|
||||
ThreatSeverity::Medium => "medium",
|
||||
ThreatSeverity::High => "high",
|
||||
ThreatSeverity::Critical => "critical",
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use common::hivemind::IoC;
|
||||
|
||||
fn sample_vioc() -> VerifiedIoC {
|
||||
VerifiedIoC {
|
||||
ioc: IoC {
|
||||
ioc_type: 0,
|
||||
severity: 3,
|
||||
ip: 0xC0A80001,
|
||||
ja4: Some("t13d1516h2_8daaf6152771_e5627efa2ab1".to_string()),
|
||||
entropy_score: Some(7500),
|
||||
description: "Malicious IP".to_string(),
|
||||
first_seen: 1700000000,
|
||||
confirmations: 3,
|
||||
zkp_proof: Vec::new(),
|
||||
},
|
||||
verified_at: 1700001000,
|
||||
stix_id: "indicator--aabbccdd-1122-3344-5566-778899aabbcc".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn splunk_event_fields() {
|
||||
let vioc = sample_vioc();
|
||||
let event = ioc_to_splunk(&vioc);
|
||||
|
||||
assert_eq!(event.time, 1700001000);
|
||||
assert_eq!(event.source, "hivemind");
|
||||
assert_eq!(event.sourcetype, hivemind::SPLUNK_SOURCETYPE);
|
||||
assert_eq!(event.event.ioc_type, "malicious_ip");
|
||||
assert_eq!(event.event.severity, "high");
|
||||
assert_eq!(event.event.src_ip, "192.168.0.1");
|
||||
assert_eq!(event.event.confirmations, 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn splunk_severity_mapping() {
|
||||
assert_eq!(severity_label(ThreatSeverity::Info), "info");
|
||||
assert_eq!(severity_label(ThreatSeverity::Critical), "critical");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn splunk_batch() {
|
||||
let iocs = vec![sample_vioc(), sample_vioc()];
|
||||
let batch = batch_to_splunk(&iocs);
|
||||
assert_eq!(batch.len(), 2);
|
||||
}
|
||||
}
|
||||
20
hivemind-api/src/lib.rs
Executable file
20
hivemind-api/src/lib.rs
Executable file
|
|
@ -0,0 +1,20 @@
|
|||
//! HiveMind Enterprise Threat Feed API.
|
||||
//!
|
||||
//! Provides REST, STIX/TAXII 2.1, and SIEM integration endpoints
|
||||
//! for consuming verified threat intelligence from the HiveMind mesh.
|
||||
//!
|
||||
//! # Modules
|
||||
//!
|
||||
//! - `store` — In-memory verified IoC storage with time-windowed queries
|
||||
//! - `stix` — STIX 2.1 types and IoC→STIX indicator conversion
|
||||
//! - `feed` — Query parameter parsing, filtering, and pagination
|
||||
//! - `integrations` — SIEM format exporters (Splunk HEC, QRadar LEEF, CEF)
|
||||
//! - `licensing` — API key management and tier-based access control
|
||||
//! - `server` — HTTP server with request routing
|
||||
|
||||
pub mod feed;
|
||||
pub mod integrations;
|
||||
pub mod licensing;
|
||||
pub mod server;
|
||||
pub mod stix;
|
||||
pub mod store;
|
||||
197
hivemind-api/src/licensing.rs
Executable file
197
hivemind-api/src/licensing.rs
Executable file
|
|
@ -0,0 +1,197 @@
|
|||
/// API key management and tier-based access control.
|
||||
///
|
||||
/// Manages API keys for the Enterprise Threat Feed. Each key is
|
||||
/// associated with an `ApiTier` that determines access to SIEM
|
||||
/// formats, page size limits, and STIX/TAXII endpoints.
|
||||
///
|
||||
/// API keys are stored as SHA256 hashes — the raw key is never
|
||||
/// persisted after initial generation.
|
||||
use common::hivemind::{self, ApiTier};
|
||||
use ring::digest;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{Arc, RwLock};
|
||||
use tracing::{info, warn};
|
||||
|
||||
/// Thread-safe handle to the license manager.
|
||||
pub type SharedLicenseManager = Arc<RwLock<LicenseManager>>;
|
||||
|
||||
/// Manages API keys and their associated tiers.
|
||||
pub struct LicenseManager {
|
||||
/// Map from SHA256(api_key) hex → tier.
|
||||
keys: HashMap<String, ApiTier>,
|
||||
}
|
||||
|
||||
impl Default for LicenseManager {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl LicenseManager {
|
||||
/// Create a new empty license manager.
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
keys: HashMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a shared (thread-safe) handle to a new license manager.
|
||||
pub fn shared() -> SharedLicenseManager {
|
||||
Arc::new(RwLock::new(Self::new()))
|
||||
}
|
||||
|
||||
/// Register an API key with the given tier.
|
||||
///
|
||||
/// The key is immediately hashed — only the hash is stored.
|
||||
/// Returns the key hash for logging purposes.
|
||||
pub fn register_key(&mut self, raw_key: &str, tier: ApiTier) -> String {
|
||||
let hash = hash_api_key(raw_key);
|
||||
self.keys.insert(hash.clone(), tier);
|
||||
info!(
|
||||
key_hash = &hash[..16],
|
||||
?tier,
|
||||
total_keys = self.keys.len(),
|
||||
"API key registered"
|
||||
);
|
||||
hash
|
||||
}
|
||||
|
||||
/// Validate an API key and return its tier.
|
||||
///
|
||||
/// Returns `None` if the key is not registered.
|
||||
pub fn validate(&self, raw_key: &str) -> Option<ApiTier> {
|
||||
let hash = hash_api_key(raw_key);
|
||||
let result = self.keys.get(&hash).copied();
|
||||
if result.is_none() {
|
||||
warn!(
|
||||
key_hash = &hash[..16],
|
||||
"API key validation failed — unknown key"
|
||||
);
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
/// Revoke an API key.
|
||||
///
|
||||
/// Returns `true` if the key existed and was removed.
|
||||
pub fn revoke_key(&mut self, raw_key: &str) -> bool {
|
||||
let hash = hash_api_key(raw_key);
|
||||
let removed = self.keys.remove(&hash).is_some();
|
||||
if removed {
|
||||
info!(key_hash = &hash[..16], "API key revoked");
|
||||
}
|
||||
removed
|
||||
}
|
||||
|
||||
/// Total number of registered API keys.
|
||||
pub fn key_count(&self) -> usize {
|
||||
self.keys.len()
|
||||
}
|
||||
}
|
||||
|
||||
/// Compute SHA256 hash of an API key, returned as lowercase hex.
|
||||
fn hash_api_key(raw_key: &str) -> String {
|
||||
let hash = digest::digest(&digest::SHA256, raw_key.as_bytes());
|
||||
hash.as_ref()
|
||||
.iter()
|
||||
.map(|b| format!("{b:02x}"))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Extract an API key from an HTTP Authorization header value.
|
||||
///
|
||||
/// Expects format: `Bearer <api-key>`
|
||||
/// Returns `None` if the header is missing or malformed.
|
||||
pub fn extract_bearer_token(auth_header: Option<&str>) -> Option<&str> {
|
||||
let header = auth_header?;
|
||||
let stripped = header.strip_prefix("Bearer ")?;
|
||||
let token = stripped.trim();
|
||||
if token.is_empty() {
|
||||
return None;
|
||||
}
|
||||
// SECURITY: Enforce maximum token length to prevent DoS via huge headers
|
||||
if token.len() > hivemind::API_KEY_LENGTH * 2 + 16 {
|
||||
return None;
|
||||
}
|
||||
Some(token)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn register_and_validate() {
|
||||
let mut mgr = LicenseManager::new();
|
||||
mgr.register_key("test-key-123", ApiTier::Enterprise);
|
||||
|
||||
let tier = mgr.validate("test-key-123");
|
||||
assert_eq!(tier, Some(ApiTier::Enterprise));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_key_returns_none() {
|
||||
let mgr = LicenseManager::new();
|
||||
assert_eq!(mgr.validate("nonexistent"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn revoke_key() {
|
||||
let mut mgr = LicenseManager::new();
|
||||
mgr.register_key("revoke-me", ApiTier::Free);
|
||||
assert!(mgr.validate("revoke-me").is_some());
|
||||
|
||||
assert!(mgr.revoke_key("revoke-me"));
|
||||
assert!(mgr.validate("revoke-me").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn different_tiers() {
|
||||
let mut mgr = LicenseManager::new();
|
||||
mgr.register_key("free-key", ApiTier::Free);
|
||||
mgr.register_key("enterprise-key", ApiTier::Enterprise);
|
||||
mgr.register_key("ns-key", ApiTier::NationalSecurity);
|
||||
|
||||
assert_eq!(mgr.validate("free-key"), Some(ApiTier::Free));
|
||||
assert_eq!(mgr.validate("enterprise-key"), Some(ApiTier::Enterprise));
|
||||
assert_eq!(
|
||||
mgr.validate("ns-key"),
|
||||
Some(ApiTier::NationalSecurity)
|
||||
);
|
||||
assert_eq!(mgr.key_count(), 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn key_hash_is_deterministic() {
|
||||
let h1 = hash_api_key("same-key");
|
||||
let h2 = hash_api_key("same-key");
|
||||
assert_eq!(h1, h2);
|
||||
assert_eq!(h1.len(), 64); // SHA256 hex = 64 chars
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_bearer_valid() {
|
||||
assert_eq!(
|
||||
extract_bearer_token(Some("Bearer my-api-key")),
|
||||
Some("my-api-key")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_bearer_missing() {
|
||||
assert_eq!(extract_bearer_token(None), None);
|
||||
assert_eq!(extract_bearer_token(Some("")), None);
|
||||
assert_eq!(extract_bearer_token(Some("Basic abc")), None);
|
||||
assert_eq!(extract_bearer_token(Some("Bearer ")), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_access_control() {
|
||||
assert!(!ApiTier::Free.can_access_siem());
|
||||
assert!(ApiTier::Enterprise.can_access_siem());
|
||||
assert!(ApiTier::NationalSecurity.can_access_taxii());
|
||||
|
||||
assert_eq!(ApiTier::Free.max_page_size(), 50);
|
||||
assert_eq!(ApiTier::Enterprise.max_page_size(), 1000);
|
||||
}
|
||||
}
|
||||
71
hivemind-api/src/main.rs
Executable file
71
hivemind-api/src/main.rs
Executable file
|
|
@ -0,0 +1,71 @@
|
|||
/// HiveMind Enterprise Threat Feed API — entry point.
|
||||
///
|
||||
/// Starts the HTTP server with configured address, initializes the
|
||||
/// in-memory IoC store and license manager, and serves threat feed
|
||||
/// endpoints.
|
||||
///
|
||||
/// # Usage
|
||||
///
|
||||
/// ```sh
|
||||
/// hivemind-api
|
||||
/// ```
|
||||
///
|
||||
/// # Configuration
|
||||
///
|
||||
/// Currently reads from defaults. Production configuration will
|
||||
/// come from a TOML file via the `common` crate config layer.
|
||||
use std::net::SocketAddr;
|
||||
|
||||
use common::hivemind;
|
||||
use tracing::info;
|
||||
use tracing_subscriber::EnvFilter;
|
||||
|
||||
use hivemind_api::licensing::LicenseManager;
|
||||
use hivemind_api::server::{self, HivemindCounters};
|
||||
use hivemind_api::store::ThreatFeedStore;
|
||||
|
||||
#[tokio::main(flavor = "current_thread")]
|
||||
async fn main() -> anyhow::Result<()> {
|
||||
// Initialize tracing
|
||||
tracing_subscriber::fmt()
|
||||
.with_env_filter(
|
||||
EnvFilter::try_from_default_env()
|
||||
.unwrap_or_else(|_| EnvFilter::new("info")),
|
||||
)
|
||||
.compact()
|
||||
.init();
|
||||
|
||||
info!(
|
||||
version = hivemind::SIEM_VERSION,
|
||||
"Starting HiveMind Enterprise Threat Feed API"
|
||||
);
|
||||
|
||||
// Initialize shared state
|
||||
let store = ThreatFeedStore::shared();
|
||||
let licensing = LicenseManager::shared();
|
||||
let counters = std::sync::Arc::new(HivemindCounters::default());
|
||||
|
||||
// Load bootstrap API keys from environment (if any)
|
||||
// SECURITY: Keys from env vars, never hardcoded
|
||||
if let Ok(key) = std::env::var("HIVEMIND_API_KEY_ENTERPRISE") {
|
||||
licensing
|
||||
.write()
|
||||
.expect("licensing lock not poisoned")
|
||||
.register_key(&key, hivemind::ApiTier::Enterprise);
|
||||
}
|
||||
if let Ok(key) = std::env::var("HIVEMIND_API_KEY_NS") {
|
||||
licensing
|
||||
.write()
|
||||
.expect("licensing lock not poisoned")
|
||||
.register_key(&key, hivemind::ApiTier::NationalSecurity);
|
||||
}
|
||||
|
||||
let addr = SocketAddr::from((
|
||||
hivemind::API_DEFAULT_ADDR
|
||||
.parse::<std::net::Ipv4Addr>()
|
||||
.expect("valid default bind address"),
|
||||
hivemind::API_DEFAULT_PORT,
|
||||
));
|
||||
|
||||
server::run(addr, store, licensing, counters).await
|
||||
}
|
||||
489
hivemind-api/src/server.rs
Executable file
489
hivemind-api/src/server.rs
Executable file
|
|
@ -0,0 +1,489 @@
|
|||
/// HTTP server with request routing and response formatting.
|
||||
///
|
||||
/// Implements a hyper 1.x HTTP server with manual path-based routing.
|
||||
/// All endpoints require API key authentication via `Authorization: Bearer <key>`
|
||||
/// header, except for the TAXII discovery endpoint.
|
||||
///
|
||||
/// # Endpoints
|
||||
///
|
||||
/// | Path | Description | Tier |
|
||||
/// |------|-------------|------|
|
||||
/// | `GET /taxii2/` | TAXII 2.1 API root discovery | Any |
|
||||
/// | `GET /taxii2/collections/` | List TAXII collections | Enterprise+ |
|
||||
/// | `GET /taxii2/collections/{id}/objects/` | STIX objects | Enterprise+ |
|
||||
/// | `GET /api/v1/feed` | JSON feed of verified IoCs | Any |
|
||||
/// | `GET /api/v1/feed/stix` | STIX 2.1 bundle | Enterprise+ |
|
||||
/// | `GET /api/v1/feed/splunk` | Splunk HEC format | Enterprise+ |
|
||||
/// | `GET /api/v1/feed/qradar` | QRadar LEEF format | Enterprise+ |
|
||||
/// | `GET /api/v1/feed/cef` | CEF format | Enterprise+ |
|
||||
/// | `GET /api/v1/stats` | Feed statistics | Any |
|
||||
use std::convert::Infallible;
|
||||
use std::net::SocketAddr;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::sync::Arc;
|
||||
|
||||
use common::hivemind::{self, ApiTier};
|
||||
use http_body_util::{BodyExt, Full};
|
||||
use hyper::body::Bytes;
|
||||
use hyper::server::conn::http1;
|
||||
use hyper::service::service_fn;
|
||||
use hyper::{Method, Request, Response, StatusCode};
|
||||
use hyper_util::rt::TokioIo;
|
||||
use tokio::net::TcpListener;
|
||||
use tracing::{error, info, warn};
|
||||
|
||||
use crate::feed;
|
||||
use crate::integrations::{cef, qradar, splunk};
|
||||
use crate::licensing::{self, SharedLicenseManager};
|
||||
use crate::stix;
|
||||
use crate::store::SharedStore;
|
||||
|
||||
/// Live counters pushed by blackwall, hivemind, and enterprise daemons (optional)
|
||||
/// via `POST /push`. Each daemon pushes only its own fields.
|
||||
#[derive(Default)]
|
||||
pub struct HivemindCounters {
|
||||
// eBPF/XDP counters (pushed by blackwall daemon)
|
||||
pub packets_total: AtomicU64,
|
||||
pub packets_passed: AtomicU64,
|
||||
pub packets_dropped: AtomicU64,
|
||||
pub anomalies_sent: AtomicU64,
|
||||
|
||||
// P2P mesh counters (pushed by hivemind daemon)
|
||||
pub peer_count: AtomicU64,
|
||||
pub iocs_shared_p2p: AtomicU64,
|
||||
pub avg_reputation_x100: AtomicU64,
|
||||
pub messages_total: AtomicU64,
|
||||
|
||||
// A2A counters (pushed by enterprise module when active)
|
||||
pub a2a_jwts_verified: AtomicU64,
|
||||
pub a2a_violations: AtomicU64,
|
||||
pub a2a_injections: AtomicU64,
|
||||
}
|
||||
|
||||
pub type SharedCounters = Arc<HivemindCounters>;
|
||||
|
||||
/// Delta payload for `POST /push`.
|
||||
///
|
||||
/// All fields are optional so each daemon can push only its own counters
|
||||
/// without zeroing out counters owned by other daemons.
|
||||
#[derive(serde::Deserialize)]
|
||||
struct CounterDelta {
|
||||
// eBPF counters (from blackwall)
|
||||
packets_total: Option<u64>,
|
||||
packets_passed: Option<u64>,
|
||||
packets_dropped: Option<u64>,
|
||||
anomalies_sent: Option<u64>,
|
||||
|
||||
// P2P counters (from hivemind)
|
||||
peer_count: Option<u64>,
|
||||
iocs_shared_p2p: Option<u64>,
|
||||
avg_reputation_x100: Option<u64>,
|
||||
messages_total: Option<u64>,
|
||||
|
||||
// A2A counters (from enterprise module)
|
||||
a2a_jwts_verified: Option<u64>,
|
||||
a2a_violations: Option<u64>,
|
||||
a2a_injections: Option<u64>,
|
||||
}
|
||||
|
||||
/// Start the HTTP server and listen for connections.
|
||||
///
|
||||
/// This function runs forever (until the process is terminated).
|
||||
/// Each incoming connection spawns a new task for HTTP/1.1 handling.
|
||||
pub async fn run(
|
||||
addr: SocketAddr,
|
||||
store: SharedStore,
|
||||
licensing: SharedLicenseManager,
|
||||
counters: SharedCounters,
|
||||
) -> anyhow::Result<()> {
|
||||
let listener = TcpListener::bind(addr).await?;
|
||||
info!(%addr, "Enterprise Threat Feed API listening");
|
||||
|
||||
loop {
|
||||
let (stream, peer) = listener.accept().await?;
|
||||
let io = TokioIo::new(stream);
|
||||
let store = store.clone();
|
||||
let licensing = licensing.clone();
|
||||
let counters = counters.clone();
|
||||
|
||||
tokio::task::spawn(async move {
|
||||
let service = service_fn(move |req| {
|
||||
let store = store.clone();
|
||||
let licensing = licensing.clone();
|
||||
let counters = counters.clone();
|
||||
async move { handle_request(req, store, licensing, counters, peer).await }
|
||||
});
|
||||
|
||||
if let Err(e) = http1::Builder::new()
|
||||
.serve_connection(io, service)
|
||||
.await
|
||||
{
|
||||
error!(peer = %peer, error = %e, "HTTP connection error");
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/// Route an HTTP request to the appropriate handler.
|
||||
async fn handle_request(
|
||||
req: Request<hyper::body::Incoming>,
|
||||
store: SharedStore,
|
||||
licensing: SharedLicenseManager,
|
||||
counters: SharedCounters,
|
||||
peer: SocketAddr,
|
||||
) -> Result<Response<Full<Bytes>>, Infallible> {
|
||||
let method = req.method().clone();
|
||||
let path = req.uri().path().to_string();
|
||||
let query = req.uri().query().map(|q| q.to_string());
|
||||
|
||||
// Extract API key
|
||||
let auth_header = req
|
||||
.headers()
|
||||
.get("authorization")
|
||||
.and_then(|v| v.to_str().ok());
|
||||
let token = licensing::extract_bearer_token(auth_header);
|
||||
let had_token = token.is_some();
|
||||
|
||||
// Validate API key
|
||||
let tier = match token {
|
||||
Some(key) => {
|
||||
let mgr = licensing
|
||||
.read()
|
||||
.expect("licensing lock not poisoned");
|
||||
mgr.validate(key)
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
|
||||
info!(
|
||||
%peer,
|
||||
%method,
|
||||
path = %path,
|
||||
authenticated = tier.is_some(),
|
||||
"Request received"
|
||||
);
|
||||
|
||||
// Route based on method + path
|
||||
let response = match (&method, path.as_str()) {
|
||||
// TAXII 2.1 endpoints
|
||||
(&Method::GET, "/taxii2/") => handle_taxii_discovery(),
|
||||
|
||||
(&Method::GET, "/taxii2/collections/") => {
|
||||
require_taxii(tier, had_token, handle_taxii_collections)
|
||||
}
|
||||
|
||||
(&Method::GET, p) if is_taxii_objects_path(p) => {
|
||||
require_taxii(tier, had_token, || {
|
||||
let store = store
|
||||
.read()
|
||||
.expect("store lock not poisoned");
|
||||
let max_page = tier.map_or(50, |t| t.max_page_size());
|
||||
let params = feed::parse_query_params(query.as_deref(), max_page);
|
||||
let result = store.query(¶ms);
|
||||
let bundle = stix::build_bundle(&result.items);
|
||||
json_response(StatusCode::OK, hivemind::STIX_CONTENT_TYPE, &bundle)
|
||||
})
|
||||
}
|
||||
|
||||
// Custom REST endpoints
|
||||
(&Method::GET, "/api/v1/feed") => {
|
||||
let effective_tier = tier.unwrap_or(ApiTier::Free);
|
||||
let store = store
|
||||
.read()
|
||||
.expect("store lock not poisoned");
|
||||
let params = feed::parse_query_params(
|
||||
query.as_deref(),
|
||||
effective_tier.max_page_size(),
|
||||
);
|
||||
let result = store.query(¶ms);
|
||||
json_response(StatusCode::OK, "application/json", &result)
|
||||
}
|
||||
|
||||
(&Method::GET, "/api/v1/feed/stix") => {
|
||||
require_taxii(tier, had_token, || {
|
||||
let store = store
|
||||
.read()
|
||||
.expect("store lock not poisoned");
|
||||
let max_page = tier.map_or(50, |t| t.max_page_size());
|
||||
let params = feed::parse_query_params(query.as_deref(), max_page);
|
||||
let result = store.query(¶ms);
|
||||
let bundle = stix::build_bundle(&result.items);
|
||||
json_response(StatusCode::OK, hivemind::STIX_CONTENT_TYPE, &bundle)
|
||||
})
|
||||
}
|
||||
|
||||
(&Method::GET, "/api/v1/feed/splunk") => {
|
||||
require_siem(tier, had_token, || {
|
||||
let store = store
|
||||
.read()
|
||||
.expect("store lock not poisoned");
|
||||
let max_page = tier.map_or(50, |t| t.max_page_size());
|
||||
let params = feed::parse_query_params(query.as_deref(), max_page);
|
||||
let result = store.query(¶ms);
|
||||
let events = splunk::batch_to_splunk(&result.items);
|
||||
json_response(StatusCode::OK, "application/json", &events)
|
||||
})
|
||||
}
|
||||
|
||||
(&Method::GET, "/api/v1/feed/qradar") => {
|
||||
require_siem(tier, had_token, || {
|
||||
let store = store
|
||||
.read()
|
||||
.expect("store lock not poisoned");
|
||||
let max_page = tier.map_or(50, |t| t.max_page_size());
|
||||
let params = feed::parse_query_params(query.as_deref(), max_page);
|
||||
let result = store.query(¶ms);
|
||||
text_response(
|
||||
StatusCode::OK,
|
||||
"text/plain",
|
||||
&qradar::batch_to_leef(&result.items),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
(&Method::GET, "/api/v1/feed/cef") => {
|
||||
require_siem(tier, had_token, || {
|
||||
let store = store
|
||||
.read()
|
||||
.expect("store lock not poisoned");
|
||||
let max_page = tier.map_or(50, |t| t.max_page_size());
|
||||
let params = feed::parse_query_params(query.as_deref(), max_page);
|
||||
let result = store.query(¶ms);
|
||||
text_response(
|
||||
StatusCode::OK,
|
||||
"text/plain",
|
||||
&cef::batch_to_cef(&result.items),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
(&Method::GET, "/api/v1/stats") => {
|
||||
let store = store
|
||||
.read()
|
||||
.expect("store lock not poisoned");
|
||||
let stats = feed::compute_stats(&store);
|
||||
json_response(StatusCode::OK, "application/json", &stats)
|
||||
}
|
||||
|
||||
// Dashboard mesh stats endpoint (no auth required)
|
||||
(&Method::GET, "/stats") => {
|
||||
let store = store
|
||||
.read()
|
||||
.expect("store lock not poisoned");
|
||||
let mesh = feed::compute_mesh_stats(&store, &counters);
|
||||
json_response(StatusCode::OK, "application/json", &mesh)
|
||||
}
|
||||
|
||||
// Internal metrics push from blackwall daemon (localhost only)
|
||||
(&Method::POST, "/push") => {
|
||||
// SECURITY: only accept from loopback
|
||||
if !peer.ip().is_loopback() {
|
||||
warn!(%peer, "rejected /push from non-loopback");
|
||||
return Ok(error_response(StatusCode::FORBIDDEN, "Forbidden"));
|
||||
}
|
||||
let body_bytes = match req.collect().await {
|
||||
Ok(b) => b.to_bytes(),
|
||||
Err(_) => return Ok(error_response(StatusCode::BAD_REQUEST, "bad body")),
|
||||
};
|
||||
match serde_json::from_slice::<CounterDelta>(&body_bytes) {
|
||||
Ok(delta) => {
|
||||
// eBPF counters (from blackwall)
|
||||
if let Some(v) = delta.packets_total {
|
||||
counters.packets_total.store(v, Ordering::Relaxed);
|
||||
}
|
||||
if let Some(v) = delta.packets_passed {
|
||||
counters.packets_passed.store(v, Ordering::Relaxed);
|
||||
}
|
||||
if let Some(v) = delta.packets_dropped {
|
||||
counters.packets_dropped.store(v, Ordering::Relaxed);
|
||||
}
|
||||
if let Some(v) = delta.anomalies_sent {
|
||||
counters.anomalies_sent.store(v, Ordering::Relaxed);
|
||||
}
|
||||
// P2P counters (from hivemind)
|
||||
if let Some(v) = delta.peer_count {
|
||||
counters.peer_count.store(v, Ordering::Relaxed);
|
||||
}
|
||||
if let Some(v) = delta.iocs_shared_p2p {
|
||||
counters.iocs_shared_p2p.store(v, Ordering::Relaxed);
|
||||
}
|
||||
if let Some(v) = delta.avg_reputation_x100 {
|
||||
counters.avg_reputation_x100.store(v, Ordering::Relaxed);
|
||||
}
|
||||
if let Some(v) = delta.messages_total {
|
||||
counters.messages_total.store(v, Ordering::Relaxed);
|
||||
}
|
||||
// A2A counters (from enterprise module)
|
||||
if let Some(v) = delta.a2a_jwts_verified {
|
||||
counters.a2a_jwts_verified.store(v, Ordering::Relaxed);
|
||||
}
|
||||
if let Some(v) = delta.a2a_violations {
|
||||
counters.a2a_violations.store(v, Ordering::Relaxed);
|
||||
}
|
||||
if let Some(v) = delta.a2a_injections {
|
||||
counters.a2a_injections.store(v, Ordering::Relaxed);
|
||||
}
|
||||
json_response(StatusCode::OK, "application/json", &serde_json::json!({"ok": true}))
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(%e, "failed to parse /push payload");
|
||||
error_response(StatusCode::BAD_REQUEST, "invalid JSON")
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
warn!(%method, path = %path, "Unknown endpoint");
|
||||
error_response(StatusCode::NOT_FOUND, "Not found")
|
||||
}
|
||||
};
|
||||
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
// --- TAXII endpoint handlers ---
|
||||
|
||||
/// Handle TAXII 2.1 API root discovery (no auth required).
|
||||
fn handle_taxii_discovery() -> Response<Full<Bytes>> {
|
||||
let discovery = stix::discovery_response();
|
||||
json_response(StatusCode::OK, hivemind::TAXII_CONTENT_TYPE, &discovery)
|
||||
}
|
||||
|
||||
/// Handle TAXII 2.1 collection listing.
|
||||
fn handle_taxii_collections() -> Response<Full<Bytes>> {
|
||||
let collections = vec![stix::default_collection()];
|
||||
let wrapper = serde_json::json!({ "collections": collections });
|
||||
json_response(StatusCode::OK, hivemind::TAXII_CONTENT_TYPE, &wrapper)
|
||||
}
|
||||
|
||||
// --- Access control helpers ---
|
||||
|
||||
/// Require Enterprise+ tier for TAXII endpoints.
|
||||
fn require_taxii<F>(tier: Option<ApiTier>, had_token: bool, f: F) -> Response<Full<Bytes>>
|
||||
where
|
||||
F: FnOnce() -> Response<Full<Bytes>>,
|
||||
{
|
||||
match tier {
|
||||
Some(t) if t.can_access_taxii() => f(),
|
||||
Some(_) => error_response(
|
||||
StatusCode::FORBIDDEN,
|
||||
"TAXII endpoints require Enterprise or NationalSecurity tier",
|
||||
),
|
||||
None if had_token => error_response(
|
||||
StatusCode::UNAUTHORIZED,
|
||||
"Invalid API key",
|
||||
),
|
||||
None => error_response(
|
||||
StatusCode::UNAUTHORIZED,
|
||||
"Authorization header with Bearer token required",
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
/// Require Enterprise+ tier for SIEM integration endpoints.
|
||||
fn require_siem<F>(tier: Option<ApiTier>, had_token: bool, f: F) -> Response<Full<Bytes>>
|
||||
where
|
||||
F: FnOnce() -> Response<Full<Bytes>>,
|
||||
{
|
||||
match tier {
|
||||
Some(t) if t.can_access_siem() => f(),
|
||||
Some(_) => error_response(
|
||||
StatusCode::FORBIDDEN,
|
||||
"SIEM integration endpoints require Enterprise or NationalSecurity tier",
|
||||
),
|
||||
None if had_token => error_response(
|
||||
StatusCode::UNAUTHORIZED,
|
||||
"Invalid API key",
|
||||
),
|
||||
None => error_response(
|
||||
StatusCode::UNAUTHORIZED,
|
||||
"Authorization header with Bearer token required",
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
// --- Path matching ---
|
||||
|
||||
/// Check if a path matches the TAXII collection objects pattern.
|
||||
///
|
||||
/// Pattern: `/taxii2/collections/<id>/objects/`
|
||||
fn is_taxii_objects_path(path: &str) -> bool {
|
||||
let Some(rest) = path.strip_prefix("/taxii2/collections/") else {
|
||||
return false;
|
||||
};
|
||||
rest.ends_with("/objects/") && rest.len() > "/objects/".len()
|
||||
}
|
||||
|
||||
// --- Response builders ---
|
||||
|
||||
/// Build a JSON response with the given status and content type.
|
||||
fn json_response<T: serde::Serialize>(
|
||||
status: StatusCode,
|
||||
content_type: &str,
|
||||
body: &T,
|
||||
) -> Response<Full<Bytes>> {
|
||||
let json = serde_json::to_string(body).unwrap_or_else(|e| {
|
||||
format!("{{\"error\":\"serialization failed: {e}\"}}")
|
||||
});
|
||||
Response::builder()
|
||||
.status(status)
|
||||
.header("content-type", content_type)
|
||||
.header("x-hivemind-version", hivemind::SIEM_VERSION)
|
||||
.body(Full::new(Bytes::from(json)))
|
||||
.expect("building response with valid parameters")
|
||||
}
|
||||
|
||||
/// Build a plain-text response.
|
||||
fn text_response(
|
||||
status: StatusCode,
|
||||
content_type: &str,
|
||||
body: &str,
|
||||
) -> Response<Full<Bytes>> {
|
||||
Response::builder()
|
||||
.status(status)
|
||||
.header("content-type", content_type)
|
||||
.header("x-hivemind-version", hivemind::SIEM_VERSION)
|
||||
.body(Full::new(Bytes::from(body.to_owned())))
|
||||
.expect("building response with valid parameters")
|
||||
}
|
||||
|
||||
/// Build a JSON error response.
|
||||
fn error_response(status: StatusCode, message: &str) -> Response<Full<Bytes>> {
|
||||
let body = serde_json::json!({
|
||||
"error": message,
|
||||
"status": status.as_u16(),
|
||||
});
|
||||
json_response(status, "application/json", &body)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn taxii_objects_path_matching() {
|
||||
assert!(is_taxii_objects_path(
|
||||
"/taxii2/collections/hivemind-threat-feed-v1/objects/"
|
||||
));
|
||||
assert!(!is_taxii_objects_path("/taxii2/collections/"));
|
||||
assert!(!is_taxii_objects_path("/taxii2/collections/objects/"));
|
||||
assert!(!is_taxii_objects_path("/api/v1/feed"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn error_response_format() {
|
||||
let resp = error_response(StatusCode::UNAUTHORIZED, "test error");
|
||||
assert_eq!(resp.status(), StatusCode::UNAUTHORIZED);
|
||||
let ct = resp.headers().get("content-type").expect("has content-type");
|
||||
assert_eq!(ct, "application/json");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn json_response_has_version_header() {
|
||||
let resp = json_response(StatusCode::OK, "application/json", &"hello");
|
||||
let ver = resp
|
||||
.headers()
|
||||
.get("x-hivemind-version")
|
||||
.expect("has version");
|
||||
assert_eq!(ver, hivemind::SIEM_VERSION);
|
||||
}
|
||||
}
|
||||
328
hivemind-api/src/stix.rs
Executable file
328
hivemind-api/src/stix.rs
Executable file
|
|
@ -0,0 +1,328 @@
|
|||
/// STIX 2.1 types and IoC-to-STIX conversion.
|
||||
///
|
||||
/// Implements core STIX Structured Threat Information Expression objects
|
||||
/// for the Enterprise Threat Feed API. Converts HiveMind IoCs to
|
||||
/// STIX Indicator SDOs within STIX Bundles.
|
||||
///
|
||||
/// Reference: <https://docs.oasis-open.org/cti/stix/v2.1/os/stix-v2.1-os.html>
|
||||
use common::hivemind::{self, IoCType, ThreatSeverity};
|
||||
use serde::Serialize;
|
||||
|
||||
use crate::store::{ip_to_string, unix_to_iso8601, VerifiedIoC};
|
||||
|
||||
/// STIX 2.1 Bundle — top-level container for STIX objects.
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
pub struct StixBundle {
|
||||
/// Always "bundle".
|
||||
#[serde(rename = "type")]
|
||||
pub object_type: &'static str,
|
||||
/// Deterministic bundle ID.
|
||||
pub id: String,
|
||||
/// List of STIX objects.
|
||||
pub objects: Vec<StixIndicator>,
|
||||
}
|
||||
|
||||
/// STIX 2.1 Indicator SDO — represents an IoC observation.
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
pub struct StixIndicator {
|
||||
/// Always "indicator".
|
||||
#[serde(rename = "type")]
|
||||
pub object_type: &'static str,
|
||||
/// STIX spec version.
|
||||
pub spec_version: &'static str,
|
||||
/// Deterministic STIX ID (from store).
|
||||
pub id: String,
|
||||
/// ISO 8601 creation timestamp.
|
||||
pub created: String,
|
||||
/// ISO 8601 modification timestamp.
|
||||
pub modified: String,
|
||||
/// Human-readable indicator name.
|
||||
pub name: String,
|
||||
/// STIX pattern expression.
|
||||
pub pattern: String,
|
||||
/// Pattern language (always "stix").
|
||||
pub pattern_type: &'static str,
|
||||
/// When this indicator becomes valid.
|
||||
pub valid_from: String,
|
||||
/// Confidence score (0-100).
|
||||
pub confidence: u8,
|
||||
/// Indicator type labels.
|
||||
pub indicator_types: Vec<&'static str>,
|
||||
/// Descriptive labels.
|
||||
pub labels: Vec<String>,
|
||||
}
|
||||
|
||||
/// TAXII 2.1 Collection resource.
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
pub struct TaxiiCollection {
|
||||
/// Collection identifier.
|
||||
pub id: String,
|
||||
/// Human-readable title.
|
||||
pub title: String,
|
||||
/// Description.
|
||||
pub description: String,
|
||||
/// Whether this collection can be read.
|
||||
pub can_read: bool,
|
||||
/// Whether this collection can be written to.
|
||||
pub can_write: bool,
|
||||
/// Supported media types.
|
||||
pub media_types: Vec<&'static str>,
|
||||
}
|
||||
|
||||
/// TAXII 2.1 API Root discovery response.
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
pub struct TaxiiDiscovery {
|
||||
/// API root title.
|
||||
pub title: String,
|
||||
/// Description.
|
||||
pub description: String,
|
||||
/// Supported TAXII versions.
|
||||
pub versions: Vec<&'static str>,
|
||||
/// Maximum content length.
|
||||
pub max_content_length: usize,
|
||||
}
|
||||
|
||||
/// Convert a verified IoC to a STIX 2.1 Indicator.
|
||||
pub fn ioc_to_indicator(vioc: &VerifiedIoC) -> StixIndicator {
|
||||
let ioc = &vioc.ioc;
|
||||
let ioc_type = IoCType::from_u8(ioc.ioc_type);
|
||||
let severity = ThreatSeverity::from_u8(ioc.severity);
|
||||
|
||||
let name = build_indicator_name(ioc_type, ioc);
|
||||
let pattern = build_stix_pattern(ioc_type, ioc);
|
||||
let confidence = severity_to_confidence(severity);
|
||||
let indicator_types = ioc_type_to_stix_types(ioc_type);
|
||||
let created = unix_to_iso8601(vioc.verified_at);
|
||||
let valid_from = unix_to_iso8601(ioc.first_seen);
|
||||
|
||||
StixIndicator {
|
||||
object_type: "indicator",
|
||||
spec_version: hivemind::STIX_SPEC_VERSION,
|
||||
id: vioc.stix_id.clone(),
|
||||
created: created.clone(),
|
||||
modified: created,
|
||||
name,
|
||||
pattern,
|
||||
pattern_type: "stix",
|
||||
valid_from,
|
||||
confidence,
|
||||
indicator_types,
|
||||
labels: vec![format!("severity:{}", ioc.severity)],
|
||||
}
|
||||
}
|
||||
|
||||
/// Build a STIX bundle from a list of verified IoCs.
|
||||
pub fn build_bundle(iocs: &[VerifiedIoC]) -> StixBundle {
|
||||
let objects: Vec<StixIndicator> = iocs.iter().map(ioc_to_indicator).collect();
|
||||
|
||||
// Bundle ID: deterministic from object count + first ID
|
||||
let bundle_suffix = if let Some(first) = objects.first() {
|
||||
first.id.chars().skip(12).take(36).collect::<String>()
|
||||
} else {
|
||||
"00000000-0000-0000-0000-000000000000".to_string()
|
||||
};
|
||||
|
||||
StixBundle {
|
||||
object_type: "bundle",
|
||||
id: format!("bundle--{bundle_suffix}"),
|
||||
objects,
|
||||
}
|
||||
}
|
||||
|
||||
/// Build the default TAXII collection descriptor.
|
||||
pub fn default_collection() -> TaxiiCollection {
|
||||
TaxiiCollection {
|
||||
id: hivemind::TAXII_COLLECTION_ID.to_string(),
|
||||
title: hivemind::TAXII_COLLECTION_TITLE.to_string(),
|
||||
description: "Consensus-verified threat indicators from the HiveMind P2P mesh. \
|
||||
Each IoC has been cross-validated by at least 3 independent peers."
|
||||
.to_string(),
|
||||
can_read: true,
|
||||
can_write: false,
|
||||
media_types: vec![hivemind::STIX_CONTENT_TYPE],
|
||||
}
|
||||
}
|
||||
|
||||
/// Build the TAXII API root discovery response.
|
||||
pub fn discovery_response() -> TaxiiDiscovery {
|
||||
TaxiiDiscovery {
|
||||
title: "HiveMind Threat Feed".to_string(),
|
||||
description: "TAXII 2.1 API for the HiveMind decentralized threat intelligence mesh."
|
||||
.to_string(),
|
||||
versions: vec!["taxii-2.1"],
|
||||
max_content_length: hivemind::MAX_MESSAGE_SIZE,
|
||||
}
|
||||
}
|
||||
|
||||
/// Build a human-readable indicator name from IoC fields.
|
||||
fn build_indicator_name(ioc_type: IoCType, ioc: &common::hivemind::IoC) -> String {
|
||||
match ioc_type {
|
||||
IoCType::MaliciousIp => {
|
||||
format!("Malicious IP {}", ip_to_string(ioc.ip))
|
||||
}
|
||||
IoCType::Ja4Fingerprint => {
|
||||
let ja4 = ioc.ja4.as_deref().unwrap_or("unknown");
|
||||
format!("Malicious JA4 fingerprint {ja4}")
|
||||
}
|
||||
IoCType::EntropyAnomaly => {
|
||||
let score = ioc.entropy_score.unwrap_or(0);
|
||||
format!("High-entropy anomaly (score={score}) from {}", ip_to_string(ioc.ip))
|
||||
}
|
||||
IoCType::DnsTunnel => {
|
||||
format!("DNS tunneling from {}", ip_to_string(ioc.ip))
|
||||
}
|
||||
IoCType::BehavioralPattern => {
|
||||
format!("Behavioral anomaly from {}", ip_to_string(ioc.ip))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Build a STIX pattern expression from an IoC.
|
||||
///
|
||||
/// STIX patterns follow the STIX Patterning language:
|
||||
/// `[<object-type>:<property> = '<value>']`
|
||||
fn build_stix_pattern(ioc_type: IoCType, ioc: &common::hivemind::IoC) -> String {
|
||||
match ioc_type {
|
||||
IoCType::MaliciousIp => {
|
||||
format!("[ipv4-addr:value = '{}']", ip_to_string(ioc.ip))
|
||||
}
|
||||
IoCType::Ja4Fingerprint => {
|
||||
let ja4 = ioc.ja4.as_deref().unwrap_or("unknown");
|
||||
format!("[network-traffic:extensions.'tls-ext'.ja4 = '{ja4}']")
|
||||
}
|
||||
IoCType::EntropyAnomaly => {
|
||||
format!(
|
||||
"[network-traffic:src_ref.type = 'ipv4-addr' AND \
|
||||
network-traffic:src_ref.value = '{}']",
|
||||
ip_to_string(ioc.ip)
|
||||
)
|
||||
}
|
||||
IoCType::DnsTunnel => {
|
||||
format!(
|
||||
"[domain-name:resolves_to_refs[*].value = '{}']",
|
||||
ip_to_string(ioc.ip)
|
||||
)
|
||||
}
|
||||
IoCType::BehavioralPattern => {
|
||||
format!(
|
||||
"[network-traffic:src_ref.type = 'ipv4-addr' AND \
|
||||
network-traffic:src_ref.value = '{}']",
|
||||
ip_to_string(ioc.ip)
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Map threat severity to STIX confidence score (0-100).
|
||||
fn severity_to_confidence(severity: ThreatSeverity) -> u8 {
|
||||
match severity {
|
||||
ThreatSeverity::Info => 20,
|
||||
ThreatSeverity::Low => 40,
|
||||
ThreatSeverity::Medium => 60,
|
||||
ThreatSeverity::High => 80,
|
||||
ThreatSeverity::Critical => 95,
|
||||
}
|
||||
}
|
||||
|
||||
/// Map IoC type to STIX indicator type labels.
|
||||
fn ioc_type_to_stix_types(ioc_type: IoCType) -> Vec<&'static str> {
|
||||
match ioc_type {
|
||||
IoCType::MaliciousIp => vec!["malicious-activity", "anomalous-activity"],
|
||||
IoCType::Ja4Fingerprint => vec!["malicious-activity"],
|
||||
IoCType::EntropyAnomaly => vec!["anomalous-activity"],
|
||||
IoCType::DnsTunnel => vec!["malicious-activity", "anomalous-activity"],
|
||||
IoCType::BehavioralPattern => vec!["anomalous-activity"],
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use common::hivemind::IoC;
|
||||
|
||||
fn sample_ioc() -> IoC {
|
||||
IoC {
|
||||
ioc_type: 0, // MaliciousIp
|
||||
severity: 3, // High
|
||||
ip: 0xC0A80001,
|
||||
ja4: Some("t13d1516h2_8daaf6152771_e5627efa2ab1".to_string()),
|
||||
entropy_score: Some(7500),
|
||||
description: "Test malicious IP".to_string(),
|
||||
first_seen: 1700000000,
|
||||
confirmations: 3,
|
||||
zkp_proof: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn sample_verified() -> VerifiedIoC {
|
||||
VerifiedIoC {
|
||||
stix_id: "indicator--aabbccdd-1122-3344-5566-778899aabbcc".to_string(),
|
||||
verified_at: 1700001000,
|
||||
ioc: sample_ioc(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ioc_converts_to_indicator() {
|
||||
let vioc = sample_verified();
|
||||
let indicator = ioc_to_indicator(&vioc);
|
||||
|
||||
assert_eq!(indicator.object_type, "indicator");
|
||||
assert_eq!(indicator.spec_version, "2.1");
|
||||
assert_eq!(indicator.id, vioc.stix_id);
|
||||
assert_eq!(indicator.pattern_type, "stix");
|
||||
assert_eq!(indicator.confidence, 80); // High severity
|
||||
assert!(indicator.name.contains("192.168.0.1"));
|
||||
assert!(indicator.pattern.contains("192.168.0.1"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bundle_creation() {
|
||||
let viocs = vec![sample_verified()];
|
||||
let bundle = build_bundle(&viocs);
|
||||
|
||||
assert_eq!(bundle.object_type, "bundle");
|
||||
assert!(bundle.id.starts_with("bundle--"));
|
||||
assert_eq!(bundle.objects.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_bundle() {
|
||||
let bundle = build_bundle(&[]);
|
||||
assert_eq!(bundle.objects.len(), 0);
|
||||
assert!(bundle.id.starts_with("bundle--"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stix_patterns_by_type() {
|
||||
let ioc = sample_ioc();
|
||||
|
||||
// MaliciousIp
|
||||
let pattern = build_stix_pattern(IoCType::MaliciousIp, &ioc);
|
||||
assert!(pattern.starts_with("[ipv4-addr:value"));
|
||||
|
||||
// Ja4Fingerprint
|
||||
let mut ja4_ioc = ioc.clone();
|
||||
ja4_ioc.ioc_type = 1;
|
||||
let pattern = build_stix_pattern(IoCType::Ja4Fingerprint, &ja4_ioc);
|
||||
assert!(pattern.contains("tls-ext"));
|
||||
|
||||
// DnsTunnel
|
||||
let pattern = build_stix_pattern(IoCType::DnsTunnel, &ioc);
|
||||
assert!(pattern.contains("domain-name"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn taxii_discovery() {
|
||||
let disc = discovery_response();
|
||||
assert!(disc.versions.contains(&"taxii-2.1"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn taxii_collection() {
|
||||
let coll = default_collection();
|
||||
assert_eq!(coll.id, hivemind::TAXII_COLLECTION_ID);
|
||||
assert!(coll.can_read);
|
||||
assert!(!coll.can_write);
|
||||
}
|
||||
}
|
||||
351
hivemind-api/src/store.rs
Executable file
351
hivemind-api/src/store.rs
Executable file
|
|
@ -0,0 +1,351 @@
|
|||
/// In-memory store for consensus-verified IoCs.
|
||||
///
|
||||
/// The `ThreatFeedStore` holds all IoCs that reached cross-validation
|
||||
/// consensus in the HiveMind mesh. It supports time-windowed queries,
|
||||
/// filtering by severity and type, and pagination for API responses.
|
||||
use common::hivemind::{self, IoC};
|
||||
use ring::digest;
|
||||
use std::sync::{Arc, RwLock};
|
||||
use tracing::info;
|
||||
|
||||
/// A consensus-verified IoC with feed metadata.
|
||||
#[derive(Clone, Debug, serde::Serialize)]
|
||||
pub struct VerifiedIoC {
|
||||
/// The verified IoC data.
|
||||
pub ioc: IoC,
|
||||
/// Unix timestamp when consensus was reached.
|
||||
pub verified_at: u64,
|
||||
/// Pre-computed deterministic STIX identifier.
|
||||
pub stix_id: String,
|
||||
}
|
||||
|
||||
/// Thread-safe handle to the IoC store.
|
||||
pub type SharedStore = Arc<RwLock<ThreatFeedStore>>;
|
||||
|
||||
/// In-memory storage for verified IoCs, sorted by verification time.
|
||||
pub struct ThreatFeedStore {
|
||||
/// Verified IoCs, ordered by `verified_at` ascending.
|
||||
iocs: Vec<VerifiedIoC>,
|
||||
}
|
||||
|
||||
impl Default for ThreatFeedStore {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl ThreatFeedStore {
|
||||
/// Create a new empty store.
|
||||
pub fn new() -> Self {
|
||||
Self { iocs: Vec::new() }
|
||||
}
|
||||
|
||||
/// Create a shared (thread-safe) handle to a new store.
|
||||
pub fn shared() -> SharedStore {
|
||||
Arc::new(RwLock::new(Self::new()))
|
||||
}
|
||||
|
||||
/// Insert a verified IoC into the store.
|
||||
///
|
||||
/// Computes the deterministic STIX ID from the IoC fields and
|
||||
/// inserts in sorted order by verification timestamp.
|
||||
pub fn insert(&mut self, ioc: IoC, verified_at: u64) {
|
||||
let stix_id = compute_stix_id(&ioc);
|
||||
let entry = VerifiedIoC {
|
||||
ioc,
|
||||
verified_at,
|
||||
stix_id,
|
||||
};
|
||||
|
||||
// Insert in sorted order (most entries append at the end)
|
||||
let pos = self
|
||||
.iocs
|
||||
.partition_point(|e| e.verified_at <= verified_at);
|
||||
self.iocs.insert(pos, entry);
|
||||
|
||||
info!(
|
||||
total = self.iocs.len(),
|
||||
verified_at,
|
||||
"IoC added to threat feed store"
|
||||
);
|
||||
}
|
||||
|
||||
/// Query IoCs with filtering and pagination.
|
||||
pub fn query(&self, params: &QueryParams) -> QueryResult {
|
||||
let filtered: Vec<&VerifiedIoC> = self
|
||||
.iocs
|
||||
.iter()
|
||||
.filter(|e| {
|
||||
if let Some(since) = params.since {
|
||||
if e.verified_at < since {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if let Some(min_sev) = params.min_severity {
|
||||
if e.ioc.severity < min_sev {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if let Some(ioc_type) = params.ioc_type {
|
||||
if e.ioc.ioc_type != ioc_type {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
true
|
||||
})
|
||||
.collect();
|
||||
|
||||
let total = filtered.len();
|
||||
let offset = params.offset.min(total);
|
||||
let limit = params.limit.min(hivemind::API_MAX_PAGE_SIZE);
|
||||
let end = (offset + limit).min(total);
|
||||
|
||||
let items: Vec<VerifiedIoC> = filtered[offset..end]
|
||||
.iter()
|
||||
.map(|e| (*e).clone())
|
||||
.collect();
|
||||
|
||||
QueryResult {
|
||||
items,
|
||||
total,
|
||||
offset,
|
||||
limit,
|
||||
}
|
||||
}
|
||||
|
||||
/// Total number of verified IoCs in the store.
|
||||
pub fn len(&self) -> usize {
|
||||
self.iocs.len()
|
||||
}
|
||||
|
||||
/// Whether the store is empty.
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.iocs.is_empty()
|
||||
}
|
||||
|
||||
/// Get all IoCs (for stats/internal use). Returns a slice reference.
|
||||
pub fn all(&self) -> &[VerifiedIoC] {
|
||||
&self.iocs
|
||||
}
|
||||
}
|
||||
|
||||
/// Parameters for querying the threat feed store.
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub struct QueryParams {
|
||||
/// Only return IoCs verified after this Unix timestamp.
|
||||
pub since: Option<u64>,
|
||||
/// Minimum severity level (0-4).
|
||||
pub min_severity: Option<u8>,
|
||||
/// Filter by IoC type.
|
||||
pub ioc_type: Option<u8>,
|
||||
/// Maximum items to return.
|
||||
pub limit: usize,
|
||||
/// Offset for pagination.
|
||||
pub offset: usize,
|
||||
}
|
||||
|
||||
impl QueryParams {
|
||||
/// Create default query with standard page size.
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
since: None,
|
||||
min_severity: None,
|
||||
ioc_type: None,
|
||||
limit: hivemind::API_DEFAULT_PAGE_SIZE,
|
||||
offset: 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Result of a store query with pagination metadata.
|
||||
#[derive(Clone, Debug, serde::Serialize)]
|
||||
pub struct QueryResult {
|
||||
/// Matching IoCs for the current page.
|
||||
pub items: Vec<VerifiedIoC>,
|
||||
/// Total matching IoCs (before pagination).
|
||||
pub total: usize,
|
||||
/// Current offset.
|
||||
pub offset: usize,
|
||||
/// Page size used.
|
||||
pub limit: usize,
|
||||
}
|
||||
|
||||
/// Compute a deterministic STIX identifier from IoC fields.
|
||||
///
|
||||
/// Format: `indicator--<uuid>` where UUID is derived from
|
||||
/// SHA256(ioc_type || ip || ja4 || first_seen).
|
||||
fn compute_stix_id(ioc: &IoC) -> String {
|
||||
let mut data = Vec::with_capacity(64);
|
||||
data.push(ioc.ioc_type);
|
||||
data.extend_from_slice(&ioc.ip.to_be_bytes());
|
||||
if let Some(ref ja4) = ioc.ja4 {
|
||||
data.extend_from_slice(ja4.as_bytes());
|
||||
}
|
||||
data.extend_from_slice(&ioc.first_seen.to_be_bytes());
|
||||
|
||||
let hash = digest::digest(&digest::SHA256, &data);
|
||||
let h = hash.as_ref();
|
||||
|
||||
// Format as UUID-like identifier (deterministic, reproducible)
|
||||
format!(
|
||||
"indicator--{:02x}{:02x}{:02x}{:02x}-{:02x}{:02x}-{:02x}{:02x}\
|
||||
-{:02x}{:02x}-{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}",
|
||||
h[0], h[1], h[2], h[3], h[4], h[5], h[6], h[7], h[8], h[9], h[10],
|
||||
h[11], h[12], h[13], h[14], h[15],
|
||||
)
|
||||
}
|
||||
|
||||
/// Convert an IPv4 u32 to dotted-decimal string.
|
||||
pub fn ip_to_string(ip: u32) -> String {
|
||||
let a = (ip >> 24) & 0xFF;
|
||||
let b = (ip >> 16) & 0xFF;
|
||||
let c = (ip >> 8) & 0xFF;
|
||||
let d = ip & 0xFF;
|
||||
format!("{a}.{b}.{c}.{d}")
|
||||
}
|
||||
|
||||
/// Convert a Unix timestamp to ISO 8601 format (YYYY-MM-DDTHH:MM:SSZ).
|
||||
///
|
||||
/// Uses Howard Hinnant's civil_from_days algorithm for calendar conversion.
|
||||
pub fn unix_to_iso8601(ts: u64) -> String {
|
||||
let time_of_day = ts % 86400;
|
||||
let hours = time_of_day / 3600;
|
||||
let minutes = (time_of_day % 3600) / 60;
|
||||
let seconds = time_of_day % 60;
|
||||
|
||||
let days = (ts / 86400) as i64;
|
||||
|
||||
// Howard Hinnant's civil_from_days algorithm
|
||||
let z = days + 719468;
|
||||
let era = if z >= 0 { z } else { z - 146096 } / 146097;
|
||||
let doe = (z - era * 146097) as u32;
|
||||
let yoe = (doe - doe / 1460 + doe / 36524 - doe / 146096) / 365;
|
||||
let y = yoe as i64 + era * 400;
|
||||
let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
|
||||
let mp = (5 * doy + 2) / 153;
|
||||
let d = doy - (153 * mp + 2) / 5 + 1;
|
||||
let m = if mp < 10 { mp + 3 } else { mp - 9 };
|
||||
let y = if m <= 2 { y + 1 } else { y };
|
||||
|
||||
format!("{y:04}-{m:02}-{d:02}T{hours:02}:{minutes:02}:{seconds:02}Z")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn make_ioc(ip: u32, severity: u8, ioc_type: u8) -> IoC {
|
||||
IoC {
|
||||
ioc_type,
|
||||
severity,
|
||||
ip,
|
||||
ja4: Some("t13d1516h2_8daaf6152771_e5627efa2ab1".to_string()),
|
||||
entropy_score: Some(7500),
|
||||
description: format!("Test IoC ip={ip}"),
|
||||
first_seen: 1700000000,
|
||||
confirmations: 3,
|
||||
zkp_proof: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn insert_and_query_all() {
|
||||
let mut store = ThreatFeedStore::new();
|
||||
store.insert(make_ioc(1, 3, 0), 1000);
|
||||
store.insert(make_ioc(2, 2, 1), 2000);
|
||||
store.insert(make_ioc(3, 4, 0), 3000);
|
||||
|
||||
let result = store.query(&QueryParams::new());
|
||||
assert_eq!(result.total, 3);
|
||||
assert_eq!(result.items.len(), 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn query_by_severity() {
|
||||
let mut store = ThreatFeedStore::new();
|
||||
store.insert(make_ioc(1, 1, 0), 1000);
|
||||
store.insert(make_ioc(2, 3, 0), 2000);
|
||||
store.insert(make_ioc(3, 4, 0), 3000);
|
||||
|
||||
let params = QueryParams {
|
||||
min_severity: Some(3),
|
||||
..QueryParams::new()
|
||||
};
|
||||
let result = store.query(¶ms);
|
||||
assert_eq!(result.total, 2);
|
||||
assert!(result.items.iter().all(|i| i.ioc.severity >= 3));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn query_by_type() {
|
||||
let mut store = ThreatFeedStore::new();
|
||||
store.insert(make_ioc(1, 3, 0), 1000);
|
||||
store.insert(make_ioc(2, 3, 1), 2000);
|
||||
store.insert(make_ioc(3, 3, 0), 3000);
|
||||
|
||||
let params = QueryParams {
|
||||
ioc_type: Some(1),
|
||||
..QueryParams::new()
|
||||
};
|
||||
let result = store.query(¶ms);
|
||||
assert_eq!(result.total, 1);
|
||||
assert_eq!(result.items[0].ioc.ioc_type, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn query_since_timestamp() {
|
||||
let mut store = ThreatFeedStore::new();
|
||||
store.insert(make_ioc(1, 3, 0), 1000);
|
||||
store.insert(make_ioc(2, 3, 0), 2000);
|
||||
store.insert(make_ioc(3, 3, 0), 3000);
|
||||
|
||||
let params = QueryParams {
|
||||
since: Some(2000),
|
||||
..QueryParams::new()
|
||||
};
|
||||
let result = store.query(¶ms);
|
||||
assert_eq!(result.total, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pagination() {
|
||||
let mut store = ThreatFeedStore::new();
|
||||
for i in 0..10 {
|
||||
store.insert(make_ioc(i, 3, 0), 1000 + u64::from(i));
|
||||
}
|
||||
|
||||
let params = QueryParams {
|
||||
limit: 3,
|
||||
offset: 2,
|
||||
..QueryParams::new()
|
||||
};
|
||||
let result = store.query(¶ms);
|
||||
assert_eq!(result.total, 10);
|
||||
assert_eq!(result.items.len(), 3);
|
||||
assert_eq!(result.offset, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stix_id_deterministic() {
|
||||
let ioc = make_ioc(0xC0A80001, 3, 0);
|
||||
let id1 = compute_stix_id(&ioc);
|
||||
let id2 = compute_stix_id(&ioc);
|
||||
assert_eq!(id1, id2);
|
||||
assert!(id1.starts_with("indicator--"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ip_conversion() {
|
||||
assert_eq!(ip_to_string(0xC0A80001), "192.168.0.1");
|
||||
assert_eq!(ip_to_string(0x0A000001), "10.0.0.1");
|
||||
assert_eq!(ip_to_string(0), "0.0.0.0");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn timestamp_conversion() {
|
||||
// 2023-11-14T22:13:20Z
|
||||
assert_eq!(unix_to_iso8601(1700000000), "2023-11-14T22:13:20Z");
|
||||
// Unix epoch
|
||||
assert_eq!(unix_to_iso8601(0), "1970-01-01T00:00:00Z");
|
||||
}
|
||||
}
|
||||
451
hivemind-api/tests/load_test.rs
Executable file
451
hivemind-api/tests/load_test.rs
Executable file
|
|
@ -0,0 +1,451 @@
|
|||
//! Load Test & Licensing Lockdown — Stress simulation for HiveMind Enterprise API.
|
||||
//!
|
||||
//! Spawns a real hyper server, seeds it with IoCs, then hammers it with
|
||||
//! concurrent clients measuring response latency and verifying tier-based
|
||||
//! access control enforcement.
|
||||
|
||||
use common::hivemind::{ApiTier, IoC};
|
||||
use hivemind_api::licensing::LicenseManager;
|
||||
use hivemind_api::server::{self, HivemindCounters};
|
||||
use hivemind_api::store::ThreatFeedStore;
|
||||
use std::net::SocketAddr;
|
||||
use std::time::Instant;
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use tokio::net::TcpStream;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Make a raw HTTP/1.1 GET request and return (status_code, body).
|
||||
async fn http_get(addr: SocketAddr, path: &str, bearer: Option<&str>) -> (u16, String) {
|
||||
let mut stream = TcpStream::connect(addr)
|
||||
.await
|
||||
.expect("TCP connect failed");
|
||||
|
||||
let mut request = format!(
|
||||
"GET {path} HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n"
|
||||
);
|
||||
if let Some(token) = bearer {
|
||||
request.push_str(&format!("Authorization: Bearer {token}\r\n"));
|
||||
}
|
||||
request.push_str("\r\n");
|
||||
|
||||
stream
|
||||
.write_all(request.as_bytes())
|
||||
.await
|
||||
.expect("write request failed");
|
||||
|
||||
let mut buf = Vec::with_capacity(8192);
|
||||
stream
|
||||
.read_to_end(&mut buf)
|
||||
.await
|
||||
.expect("read response failed");
|
||||
|
||||
let raw = String::from_utf8_lossy(&buf);
|
||||
|
||||
// Parse status code from "HTTP/1.1 NNN ..."
|
||||
let status = raw
|
||||
.get(9..12)
|
||||
.and_then(|s| s.parse::<u16>().ok())
|
||||
.unwrap_or(0);
|
||||
|
||||
// Split at the blank line separating headers from body
|
||||
let body = raw
|
||||
.find("\r\n\r\n")
|
||||
.map(|i| raw[i + 4..].to_string())
|
||||
.unwrap_or_default();
|
||||
|
||||
(status, body)
|
||||
}
|
||||
|
||||
/// Seed the store with `count` synthetic IoCs at 1-second intervals.
|
||||
fn seed_store(store: &mut ThreatFeedStore, count: usize) {
|
||||
let base_time = 1_700_000_000u64;
|
||||
for i in 0..count {
|
||||
let ioc = IoC {
|
||||
ioc_type: (i % 5) as u8,
|
||||
severity: ((i % 5) as u8).min(4),
|
||||
ip: 0xC6120000 + i as u32, // 198.18.x.x range
|
||||
ja4: if i % 3 == 0 {
|
||||
Some("t13d1516h2_8daaf6152771_e5627efa2ab1".into())
|
||||
} else {
|
||||
None
|
||||
},
|
||||
entropy_score: Some(5000 + (i as u32 * 100)),
|
||||
description: format!("Synthetic threat indicator #{i}"),
|
||||
first_seen: base_time + i as u64,
|
||||
confirmations: 3,
|
||||
zkp_proof: Vec::new(),
|
||||
};
|
||||
store.insert(ioc, base_time + i as u64 + 60);
|
||||
}
|
||||
}
|
||||
|
||||
/// Find a free TCP port by binding to :0 and reading the assigned port.
|
||||
async fn free_port() -> u16 {
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
||||
.await
|
||||
.expect("bind :0 failed");
|
||||
listener
|
||||
.local_addr()
|
||||
.expect("local_addr failed")
|
||||
.port()
|
||||
}
|
||||
|
||||
// ===========================================================================
|
||||
// Scenario A — API Hammering: 100 concurrent clients
|
||||
// ===========================================================================
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn scenario_a_api_hammering() {
|
||||
// Setup
|
||||
let store = ThreatFeedStore::shared();
|
||||
let licensing = LicenseManager::shared();
|
||||
|
||||
// Seed 50 IoCs
|
||||
{
|
||||
let mut s = store.write().expect("lock");
|
||||
seed_store(&mut s, 50);
|
||||
}
|
||||
|
||||
// Register an Enterprise-tier key
|
||||
let api_key = "test-enterprise-key-12345678";
|
||||
{
|
||||
let mut lm = licensing.write().expect("lock");
|
||||
lm.register_key(api_key, ApiTier::Enterprise);
|
||||
}
|
||||
|
||||
// Start server on a random port
|
||||
let port = free_port().await;
|
||||
let addr: SocketAddr = ([127, 0, 0, 1], port).into();
|
||||
let server_store = store.clone();
|
||||
let server_lic = licensing.clone();
|
||||
let counters = std::sync::Arc::new(HivemindCounters::default());
|
||||
let server_handle = tokio::task::spawn(async move {
|
||||
let _ = server::run(addr, server_store, server_lic, counters).await;
|
||||
});
|
||||
|
||||
// Give the server a moment to bind — try connecting in a quick retry loop
|
||||
let mut connected = false;
|
||||
for _ in 0..50 {
|
||||
if TcpStream::connect(addr).await.is_ok() {
|
||||
connected = true;
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
assert!(connected, "Server did not start within retry window");
|
||||
|
||||
// Define the endpoints to hit
|
||||
let endpoints = [
|
||||
"/api/v1/feed",
|
||||
"/api/v1/feed/stix",
|
||||
"/api/v1/feed/splunk",
|
||||
"/api/v1/stats",
|
||||
];
|
||||
|
||||
// Spawn 100 concurrent client tasks
|
||||
let client_count = 100;
|
||||
let mut handles = Vec::with_capacity(client_count);
|
||||
let start = Instant::now();
|
||||
|
||||
for i in 0..client_count {
|
||||
let endpoint = endpoints[i % endpoints.len()];
|
||||
let key = api_key.to_string();
|
||||
handles.push(tokio::task::spawn(async move {
|
||||
let t = Instant::now();
|
||||
let (status, body) = http_get(addr, endpoint, Some(&key)).await;
|
||||
let latency = t.elapsed();
|
||||
(i, status, body.len(), latency)
|
||||
}));
|
||||
}
|
||||
|
||||
// Collect results
|
||||
let mut total_latency = std::time::Duration::ZERO;
|
||||
let mut max_latency = std::time::Duration::ZERO;
|
||||
let mut error_count = 0;
|
||||
|
||||
for handle in handles {
|
||||
let (idx, status, body_len, latency) = handle.await.expect("task panicked");
|
||||
if status != 200 {
|
||||
error_count += 1;
|
||||
eprintln!(
|
||||
"[HAMMER] Client {idx}: HTTP {status} (body {body_len}B) in {latency:.2?}"
|
||||
);
|
||||
}
|
||||
total_latency += latency;
|
||||
if latency > max_latency {
|
||||
max_latency = latency;
|
||||
}
|
||||
}
|
||||
|
||||
let total_elapsed = start.elapsed();
|
||||
let avg_latency = total_latency / client_count as u32;
|
||||
|
||||
eprintln!(
|
||||
"[HAMMER] {client_count} clients completed in {total_elapsed:.2?}"
|
||||
);
|
||||
eprintln!(
|
||||
"[HAMMER] Avg latency: {avg_latency:.2?}, Max: {max_latency:.2?}, Errors: {error_count}"
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
error_count, 0,
|
||||
"All authenticated requests should succeed (HTTP 200)"
|
||||
);
|
||||
|
||||
// Abort the server
|
||||
server_handle.abort();
|
||||
}
|
||||
|
||||
// ===========================================================================
|
||||
// Scenario B — Licensing Lockdown: tier-based access denial
|
||||
// ===========================================================================
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn scenario_b_licensing_lockdown() {
|
||||
let store = ThreatFeedStore::shared();
|
||||
let licensing = LicenseManager::shared();
|
||||
|
||||
// Seed 10 IoCs
|
||||
{
|
||||
let mut s = store.write().expect("lock");
|
||||
seed_store(&mut s, 10);
|
||||
}
|
||||
|
||||
// Register keys at each tier
|
||||
let free_key = "free-tier-key-aaaa";
|
||||
let enterprise_key = "enterprise-tier-key-bbbb";
|
||||
let ns_key = "national-security-key-cccc";
|
||||
{
|
||||
let mut lm = licensing.write().expect("lock");
|
||||
lm.register_key(free_key, ApiTier::Free);
|
||||
lm.register_key(enterprise_key, ApiTier::Enterprise);
|
||||
lm.register_key(ns_key, ApiTier::NationalSecurity);
|
||||
}
|
||||
|
||||
let port = free_port().await;
|
||||
let addr: SocketAddr = ([127, 0, 0, 1], port).into();
|
||||
let server_store = store.clone();
|
||||
let server_lic = licensing.clone();
|
||||
let counters = std::sync::Arc::new(HivemindCounters::default());
|
||||
let server_handle = tokio::task::spawn(async move {
|
||||
let _ = server::run(addr, server_store, server_lic, counters).await;
|
||||
});
|
||||
|
||||
// Wait for server
|
||||
for _ in 0..50 {
|
||||
if TcpStream::connect(addr).await.is_ok() {
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
|
||||
// --- Free tier: /api/v1/feed should work ---
|
||||
let (status, _) = http_get(addr, "/api/v1/feed", Some(free_key)).await;
|
||||
assert_eq!(status, 200, "Free tier should access /api/v1/feed");
|
||||
|
||||
// --- Free tier: /api/v1/stats should work ---
|
||||
let (status, _) = http_get(addr, "/api/v1/stats", Some(free_key)).await;
|
||||
assert_eq!(status, 200, "Free tier should access /api/v1/stats");
|
||||
|
||||
// --- Free tier: SIEM endpoints BLOCKED ---
|
||||
let siem_paths = [
|
||||
"/api/v1/feed/splunk",
|
||||
"/api/v1/feed/qradar",
|
||||
"/api/v1/feed/cef",
|
||||
];
|
||||
for path in &siem_paths {
|
||||
let (status, _) = http_get(addr, path, Some(free_key)).await;
|
||||
assert_eq!(
|
||||
status, 403,
|
||||
"Free tier should be FORBIDDEN from {path}"
|
||||
);
|
||||
}
|
||||
|
||||
// --- Free tier: STIX/TAXII endpoints BLOCKED ---
|
||||
let taxii_paths = [
|
||||
"/api/v1/feed/stix",
|
||||
"/taxii2/collections/",
|
||||
];
|
||||
for path in &taxii_paths {
|
||||
let (status, _) = http_get(addr, path, Some(free_key)).await;
|
||||
assert_eq!(
|
||||
status, 403,
|
||||
"Free tier should be FORBIDDEN from {path}"
|
||||
);
|
||||
}
|
||||
|
||||
// --- No auth: should get 401 Unauthorized ---
|
||||
let (status, _) = http_get(addr, "/api/v1/feed/splunk", None).await;
|
||||
assert_eq!(
|
||||
status, 401,
|
||||
"No auth header should yield 401 Unauthorized"
|
||||
);
|
||||
|
||||
// --- Invalid key: should get denied ---
|
||||
let (status, _) = http_get(addr, "/api/v1/feed", Some("totally-bogus-key")).await;
|
||||
// /api/v1/feed allows unauthenticated access via effective_tier=Free fallback,
|
||||
// but with an invalid key, the server resolves tier to None and falls through
|
||||
// to Free default for /api/v1/feed
|
||||
assert!(
|
||||
status == 200 || status == 401,
|
||||
"/api/v1/feed with invalid key: got {status}"
|
||||
);
|
||||
|
||||
// --- Enterprise tier: SIEM endpoints ALLOWED ---
|
||||
for path in &siem_paths {
|
||||
let (status, body) = http_get(addr, path, Some(enterprise_key)).await;
|
||||
assert_eq!(
|
||||
status, 200,
|
||||
"Enterprise tier should access {path}, got {status}"
|
||||
);
|
||||
assert!(!body.is_empty(), "{path} response body should not be empty");
|
||||
}
|
||||
|
||||
// --- Enterprise tier: TAXII endpoints ALLOWED ---
|
||||
for path in &taxii_paths {
|
||||
let (status, _) = http_get(addr, path, Some(enterprise_key)).await;
|
||||
assert_eq!(
|
||||
status, 200,
|
||||
"Enterprise tier should access {path}, got {status}"
|
||||
);
|
||||
}
|
||||
|
||||
// --- NationalSecurity tier: everything ALLOWED ---
|
||||
let all_paths = [
|
||||
"/api/v1/feed",
|
||||
"/api/v1/feed/stix",
|
||||
"/api/v1/feed/splunk",
|
||||
"/api/v1/feed/qradar",
|
||||
"/api/v1/feed/cef",
|
||||
"/api/v1/stats",
|
||||
"/taxii2/",
|
||||
"/taxii2/collections/",
|
||||
];
|
||||
for path in &all_paths {
|
||||
let (status, _) = http_get(addr, path, Some(ns_key)).await;
|
||||
assert_eq!(
|
||||
status, 200,
|
||||
"NationalSecurity tier should access {path}, got {status}"
|
||||
);
|
||||
}
|
||||
|
||||
// --- Unknown endpoint: 404 ---
|
||||
let (status, _) = http_get(addr, "/api/v1/nonexistent", Some(enterprise_key)).await;
|
||||
assert_eq!(status, 404, "Unknown endpoint should yield 404");
|
||||
|
||||
eprintln!("[LOCKDOWN] All tier-based access control assertions passed");
|
||||
server_handle.abort();
|
||||
}
|
||||
|
||||
// ===========================================================================
|
||||
// Scenario C — Feed Content Integrity: verify response payloads
|
||||
// ===========================================================================
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn scenario_c_feed_content_integrity() {
|
||||
let store = ThreatFeedStore::shared();
|
||||
let licensing = LicenseManager::shared();
|
||||
|
||||
// Seed 5 IoCs
|
||||
{
|
||||
let mut s = store.write().expect("lock");
|
||||
seed_store(&mut s, 5);
|
||||
}
|
||||
|
||||
let api_key = "integrity-test-key";
|
||||
{
|
||||
let mut lm = licensing.write().expect("lock");
|
||||
lm.register_key(api_key, ApiTier::Enterprise);
|
||||
}
|
||||
|
||||
let port = free_port().await;
|
||||
let addr: SocketAddr = ([127, 0, 0, 1], port).into();
|
||||
let ss = store.clone();
|
||||
let sl = licensing.clone();
|
||||
let counters = std::sync::Arc::new(HivemindCounters::default());
|
||||
let server_handle = tokio::task::spawn(async move {
|
||||
let _ = server::run(addr, ss, sl, counters).await;
|
||||
});
|
||||
|
||||
for _ in 0..50 {
|
||||
if TcpStream::connect(addr).await.is_ok() {
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
|
||||
// --- JSON feed ---
|
||||
let (status, body) = http_get(addr, "/api/v1/feed", Some(api_key)).await;
|
||||
assert_eq!(status, 200);
|
||||
let parsed: serde_json::Value = serde_json::from_str(&body)
|
||||
.unwrap_or_else(|e| panic!("Invalid JSON in /api/v1/feed response: {e}"));
|
||||
assert!(
|
||||
parsed.get("items").is_some(),
|
||||
"Feed response should contain 'items' field"
|
||||
);
|
||||
assert!(
|
||||
parsed.get("total").is_some(),
|
||||
"Feed response should contain 'total' field"
|
||||
);
|
||||
|
||||
// --- STIX bundle ---
|
||||
let (status, body) = http_get(addr, "/api/v1/feed/stix", Some(api_key)).await;
|
||||
assert_eq!(status, 200);
|
||||
let stix: serde_json::Value = serde_json::from_str(&body)
|
||||
.unwrap_or_else(|e| panic!("Invalid STIX JSON: {e}"));
|
||||
assert_eq!(
|
||||
stix.get("type").and_then(|t| t.as_str()),
|
||||
Some("bundle"),
|
||||
"STIX response should be a bundle"
|
||||
);
|
||||
|
||||
// --- Splunk HEC ---
|
||||
let (status, body) = http_get(addr, "/api/v1/feed/splunk", Some(api_key)).await;
|
||||
assert_eq!(status, 200);
|
||||
let splunk: serde_json::Value = serde_json::from_str(&body)
|
||||
.unwrap_or_else(|e| panic!("Invalid Splunk JSON: {e}"));
|
||||
assert!(splunk.is_array(), "Splunk response should be a JSON array");
|
||||
|
||||
// --- QRadar LEEF (plain text) ---
|
||||
let (status, body) = http_get(addr, "/api/v1/feed/qradar", Some(api_key)).await;
|
||||
assert_eq!(status, 200);
|
||||
assert!(
|
||||
body.contains("LEEF:"),
|
||||
"QRadar response should contain LEEF headers"
|
||||
);
|
||||
|
||||
// --- CEF (plain text) ---
|
||||
let (status, body) = http_get(addr, "/api/v1/feed/cef", Some(api_key)).await;
|
||||
assert_eq!(status, 200);
|
||||
assert!(
|
||||
body.contains("CEF:"),
|
||||
"CEF response should contain CEF headers"
|
||||
);
|
||||
|
||||
// --- Stats ---
|
||||
let (status, body) = http_get(addr, "/api/v1/stats", Some(api_key)).await;
|
||||
assert_eq!(status, 200);
|
||||
let stats: serde_json::Value = serde_json::from_str(&body)
|
||||
.unwrap_or_else(|e| panic!("Invalid stats JSON: {e}"));
|
||||
let total = stats
|
||||
.get("total_iocs")
|
||||
.and_then(|v| v.as_u64())
|
||||
.unwrap_or(0);
|
||||
assert_eq!(total, 5, "Stats should report 5 total IoCs");
|
||||
|
||||
// --- TAXII discovery (no auth needed, but we send one) ---
|
||||
let (status, body) = http_get(addr, "/taxii2/", Some(api_key)).await;
|
||||
assert_eq!(status, 200);
|
||||
let taxii: serde_json::Value = serde_json::from_str(&body)
|
||||
.unwrap_or_else(|e| panic!("Invalid TAXII JSON: {e}"));
|
||||
assert!(
|
||||
taxii.get("title").is_some(),
|
||||
"TAXII discovery should contain title"
|
||||
);
|
||||
|
||||
eprintln!("[INTEGRITY] All 7 endpoints return well-formed responses");
|
||||
server_handle.abort();
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue