Release prep: 54 engines, self-hosted signatures, i18n, dashboard updates

This commit is contained in:
DmitrL-dev 2026-03-23 16:45:40 +10:00
parent 694e32be26
commit 41cbfd6e0a
178 changed files with 36008 additions and 399 deletions

View file

@ -0,0 +1,278 @@
package shadow_ai
import (
"fmt"
"log/slog"
"sync"
"time"
)
// --- Tiered Approval Workflow ---
// Implements §6 of the ТЗ: data classification → approval tier → SLA tracking.
// ApprovalStatus tracks the state of an approval request.
type ApprovalStatus string
const (
ApprovalPending ApprovalStatus = "pending"
ApprovalApproved ApprovalStatus = "approved"
ApprovalDenied ApprovalStatus = "denied"
ApprovalExpired ApprovalStatus = "expired"
ApprovalAutoApproved ApprovalStatus = "auto_approved"
)
// DefaultApprovalTiers defines the approval requirements per data classification.
func DefaultApprovalTiers() []ApprovalTier {
return []ApprovalTier{
{
Name: "Tier 1: Public Data",
DataClass: DataPublic,
ApprovalNeeded: nil, // Auto-approve
SLA: 0,
AutoApprove: true,
},
{
Name: "Tier 2: Internal Data",
DataClass: DataInternal,
ApprovalNeeded: []string{"manager"},
SLA: 4 * time.Hour,
AutoApprove: false,
},
{
Name: "Tier 3: Confidential Data",
DataClass: DataConfidential,
ApprovalNeeded: []string{"manager", "soc"},
SLA: 24 * time.Hour,
AutoApprove: false,
},
{
Name: "Tier 4: Critical Data",
DataClass: DataCritical,
ApprovalNeeded: []string{"ciso"},
SLA: 0, // Manual only, no auto-expire
AutoApprove: false,
},
}
}
// ApprovalEngine manages the tiered approval workflow.
type ApprovalEngine struct {
mu sync.RWMutex
tiers []ApprovalTier
requests map[string]*ApprovalRequest
logger *slog.Logger
}
// NewApprovalEngine creates an engine with default tiers.
func NewApprovalEngine() *ApprovalEngine {
return &ApprovalEngine{
tiers: DefaultApprovalTiers(),
requests: make(map[string]*ApprovalRequest),
logger: slog.Default().With("component", "shadow-ai-approvals"),
}
}
// SubmitRequest creates a new approval request based on data classification.
// Returns the request or auto-approves if the tier allows it.
func (ae *ApprovalEngine) SubmitRequest(userID, docID string, dataClass DataClassification) *ApprovalRequest {
ae.mu.Lock()
defer ae.mu.Unlock()
tier := ae.findTier(dataClass)
req := &ApprovalRequest{
ID: genApprovalID(),
DocID: docID,
UserID: userID,
Tier: tier.Name,
DataClass: dataClass,
Status: string(ApprovalPending),
CreatedAt: time.Now(),
}
// Set expiry based on SLA.
if tier.SLA > 0 {
req.ExpiresAt = req.CreatedAt.Add(tier.SLA)
}
// Auto-approve for public data.
if tier.AutoApprove {
req.Status = string(ApprovalAutoApproved)
req.ApprovedBy = "system"
req.ResolvedAt = time.Now()
ae.logger.Info("auto-approved",
"request_id", req.ID,
"user", userID,
"data_class", dataClass,
)
} else {
ae.logger.Info("approval required",
"request_id", req.ID,
"user", userID,
"data_class", dataClass,
"tier", tier.Name,
"approvers", tier.ApprovalNeeded,
)
}
ae.requests[req.ID] = req
return req
}
// Approve approves a pending request.
func (ae *ApprovalEngine) Approve(requestID, approvedBy string) error {
ae.mu.Lock()
defer ae.mu.Unlock()
req, ok := ae.requests[requestID]
if !ok {
return fmt.Errorf("request %s not found", requestID)
}
if req.Status != string(ApprovalPending) {
return fmt.Errorf("request %s is not pending (status: %s)", requestID, req.Status)
}
req.Status = string(ApprovalApproved)
req.ApprovedBy = approvedBy
req.ResolvedAt = time.Now()
ae.logger.Info("approved",
"request_id", requestID,
"approved_by", approvedBy,
)
return nil
}
// Deny denies a pending request.
func (ae *ApprovalEngine) Deny(requestID, deniedBy, reason string) error {
ae.mu.Lock()
defer ae.mu.Unlock()
req, ok := ae.requests[requestID]
if !ok {
return fmt.Errorf("request %s not found", requestID)
}
if req.Status != string(ApprovalPending) {
return fmt.Errorf("request %s is not pending (status: %s)", requestID, req.Status)
}
req.Status = string(ApprovalDenied)
req.DeniedBy = deniedBy
req.Reason = reason
req.ResolvedAt = time.Now()
ae.logger.Info("denied",
"request_id", requestID,
"denied_by", deniedBy,
"reason", reason,
)
return nil
}
// GetRequest returns an approval request by ID.
func (ae *ApprovalEngine) GetRequest(requestID string) (*ApprovalRequest, bool) {
ae.mu.RLock()
defer ae.mu.RUnlock()
req, ok := ae.requests[requestID]
if !ok {
return nil, false
}
cp := *req
return &cp, true
}
// PendingRequests returns all pending approval requests.
func (ae *ApprovalEngine) PendingRequests() []ApprovalRequest {
ae.mu.RLock()
defer ae.mu.RUnlock()
var result []ApprovalRequest
for _, req := range ae.requests {
if req.Status == string(ApprovalPending) {
result = append(result, *req)
}
}
return result
}
// ExpireOverdue marks overdue pending requests as expired.
// Returns the number of expired requests.
func (ae *ApprovalEngine) ExpireOverdue() int {
ae.mu.Lock()
defer ae.mu.Unlock()
now := time.Now()
expired := 0
for _, req := range ae.requests {
if req.Status == string(ApprovalPending) && !req.ExpiresAt.IsZero() && now.After(req.ExpiresAt) {
req.Status = string(ApprovalExpired)
req.ResolvedAt = now
expired++
ae.logger.Warn("request expired",
"request_id", req.ID,
"user", req.UserID,
"expired_at", req.ExpiresAt,
)
}
}
return expired
}
// Stats returns approval workflow statistics.
func (ae *ApprovalEngine) Stats() map[string]int {
ae.mu.RLock()
defer ae.mu.RUnlock()
stats := map[string]int{
"total": len(ae.requests),
"pending": 0,
"approved": 0,
"denied": 0,
"expired": 0,
"auto_approved": 0,
}
for _, req := range ae.requests {
switch ApprovalStatus(req.Status) {
case ApprovalPending:
stats["pending"]++
case ApprovalApproved:
stats["approved"]++
case ApprovalDenied:
stats["denied"]++
case ApprovalExpired:
stats["expired"]++
case ApprovalAutoApproved:
stats["auto_approved"]++
}
}
return stats
}
// Tiers returns the approval tier configuration.
func (ae *ApprovalEngine) Tiers() []ApprovalTier {
return ae.tiers
}
func (ae *ApprovalEngine) findTier(dataClass DataClassification) ApprovalTier {
for _, t := range ae.tiers {
if t.DataClass == dataClass {
return t
}
}
// Default to most restrictive.
return ae.tiers[len(ae.tiers)-1]
}
var approvalCounter uint64
var approvalCounterMu sync.Mutex
func genApprovalID() string {
approvalCounterMu.Lock()
approvalCounter++
id := approvalCounter
approvalCounterMu.Unlock()
return fmt.Sprintf("apr-%d-%d", time.Now().UnixMilli(), id)
}

View file

@ -0,0 +1,116 @@
package shadow_ai
import (
"time"
domsoc "github.com/syntrex/gomcp/internal/domain/soc"
)
// ShadowAICorrelationRules returns SOC correlation rules specific to Shadow AI
// detection. These integrate into the existing SOC correlation engine.
func ShadowAICorrelationRules() []domsoc.SOCCorrelationRule {
return []domsoc.SOCCorrelationRule{
{
ID: "SAI-CR-001",
Name: "Multi-Service Shadow AI",
RequiredCategories: []string{"shadow_ai_usage"},
MinEvents: 3,
TimeWindow: 10 * time.Minute,
Severity: domsoc.SeverityHigh,
KillChainPhase: "Reconnaissance",
MITREMapping: []string{"T1595"},
Description: "User accessing 3+ distinct AI services within 10 minutes. Indicates active AI tool exploration or data shopping across providers.",
},
{
ID: "SAI-CR-002",
Name: "Shadow AI + Data Exfiltration",
RequiredCategories: []string{"shadow_ai_usage", "exfiltration"},
MinEvents: 2,
TimeWindow: 15 * time.Minute,
Severity: domsoc.SeverityCritical,
KillChainPhase: "Exfiltration",
MITREMapping: []string{"T1041", "T1567"},
Description: "Shadow AI usage followed by data exfiltration attempt. Possible corporate data leakage via unauthorized AI services.",
},
{
ID: "SAI-CR-003",
Name: "Shadow AI Volume Spike",
RequiredCategories: []string{"shadow_ai_usage"},
MinEvents: 10,
TimeWindow: 1 * time.Hour,
Severity: domsoc.SeverityHigh,
KillChainPhase: "Actions on Objectives",
MITREMapping: []string{"T1048"},
Description: "10+ shadow AI events from same source within 1 hour. Indicates bulk data transfer to external AI service.",
},
{
ID: "SAI-CR-004",
Name: "Shadow AI After Hours",
RequiredCategories: []string{"shadow_ai_usage"},
MinEvents: 2,
TimeWindow: 30 * time.Minute,
Severity: domsoc.SeverityMedium,
KillChainPhase: "Persistence",
MITREMapping: []string{"T1053"},
Description: "Shadow AI usage outside business hours (detected via timestamp clustering). May indicate automated scripts or insider threat.",
},
{
ID: "SAI-CR-005",
Name: "Integration Failure Chain",
RequiredCategories: []string{"integration_health"},
MinEvents: 3,
TimeWindow: 5 * time.Minute,
Severity: domsoc.SeverityCritical,
KillChainPhase: "Defense Evasion",
MITREMapping: []string{"T1562"},
Description: "3+ integration health failures in 5 minutes. Possible attack on enforcement infrastructure to blind Shadow AI detection.",
},
{
ID: "SAI-CR-006",
Name: "Shadow AI + PII Leak",
RequiredCategories: []string{"shadow_ai_usage", "pii_leak"},
MinEvents: 2,
TimeWindow: 10 * time.Minute,
Severity: domsoc.SeverityCritical,
KillChainPhase: "Exfiltration",
MITREMapping: []string{"T1567.002"},
Description: "Shadow AI usage combined with PII leak detection. GDPR/regulatory violation in progress — immediate response required.",
},
{
ID: "SAI-CR-007",
Name: "Shadow AI Evasion Attempt",
SequenceCategories: []string{"shadow_ai_usage", "evasion"},
MinEvents: 2,
TimeWindow: 10 * time.Minute,
Severity: domsoc.SeverityHigh,
KillChainPhase: "Defense Evasion",
MITREMapping: []string{"T1090", "T1573"},
Description: "Shadow AI usage followed by evasion technique (VPN, proxy chaining, encoding). User attempting to bypass detection.",
},
{
ID: "SAI-CR-008",
Name: "Cross-Department AI Usage",
RequiredCategories: []string{"shadow_ai_usage"},
MinEvents: 5,
TimeWindow: 30 * time.Minute,
Severity: domsoc.SeverityMedium,
CrossSensor: true,
KillChainPhase: "Lateral Movement",
MITREMapping: []string{"T1021"},
Description: "Shadow AI events from 5+ distinct network segments/sensors within 30 minutes. Indicates coordinated policy circumvention or compromised credentials used across departments.",
},
// Severity trend: escalating shadow AI event severity
{
ID: "SAI-CR-009",
Name: "Shadow AI Escalation",
SeverityTrend: "ascending",
TrendCategory: "shadow_ai_usage",
MinEvents: 3,
TimeWindow: 30 * time.Minute,
Severity: domsoc.SeverityCritical,
KillChainPhase: "Exploitation",
MITREMapping: []string{"T1059"},
Description: "Ascending severity pattern in Shadow AI events: user escalating from casual browsing to bulk data uploads. Crescendo data theft in progress.",
},
}
}

View file

@ -0,0 +1,503 @@
package shadow_ai
import (
"log/slog"
"math"
"regexp"
"sort"
"strings"
"sync"
"time"
)
// --- AI Signature Database ---
// AISignatureDB contains known AI service signatures for detection.
type AISignatureDB struct {
mu sync.RWMutex
services []AIServiceInfo
domainPatterns []*domainPattern
apiKeyPatterns []*APIKeyPattern
httpSignatures []string
}
type domainPattern struct {
original string
regex *regexp.Regexp
service string
}
// APIKeyPattern defines a regex pattern for detecting AI API keys.
type APIKeyPattern struct {
Name string `json:"name"`
Pattern *regexp.Regexp `json:"-"`
Entropy float64 `json:"min_entropy"`
}
// NewAISignatureDB creates a signature database pre-loaded with known AI services.
func NewAISignatureDB() *AISignatureDB {
db := &AISignatureDB{}
db.loadDefaults()
return db
}
// loadDefaults populates the database with known AI services and patterns.
func (db *AISignatureDB) loadDefaults() {
db.services = defaultAIServices()
// Compile domain patterns.
for _, svc := range db.services {
for _, d := range svc.Domains {
pattern := domainToRegex(d)
db.domainPatterns = append(db.domainPatterns, &domainPattern{
original: d,
regex: pattern,
service: svc.Name,
})
}
}
// API key patterns.
db.apiKeyPatterns = defaultAPIKeyPatterns()
// HTTP header signatures.
db.httpSignatures = []string{
"authorization: bearer sk-", // OpenAI
"authorization: bearer ant-", // Anthropic
"x-api-key: sk-ant-", // Anthropic v2
"x-goog-api-key:", // Google AI
"authorization: bearer gsk_", // Groq
"authorization: bearer hf_", // HuggingFace
}
}
// domainToRegex converts a wildcard domain (e.g., "*.openai.com") to a regex.
func domainToRegex(domain string) *regexp.Regexp {
escaped := regexp.QuoteMeta(domain)
escaped = strings.ReplaceAll(escaped, `\*`, `[a-zA-Z0-9\-]+`)
return regexp.MustCompile("(?i)^" + escaped + "$")
}
// MatchDomain checks if a domain matches any known AI service.
// Returns the service name or empty string.
func (db *AISignatureDB) MatchDomain(domain string) string {
db.mu.RLock()
defer db.mu.RUnlock()
domain = strings.ToLower(strings.TrimSpace(domain))
for _, dp := range db.domainPatterns {
if dp.regex.MatchString(domain) {
return dp.service
}
}
return ""
}
// MatchHTTPHeaders checks if HTTP headers contain known AI service signatures.
func (db *AISignatureDB) MatchHTTPHeaders(headers map[string]string) string {
db.mu.RLock()
defer db.mu.RUnlock()
for key, value := range headers {
headerLine := strings.ToLower(key + ": " + value)
for _, sig := range db.httpSignatures {
if strings.Contains(headerLine, sig) {
return sig
}
}
}
return ""
}
// ScanForAPIKeys scans content for AI API keys.
// Returns the matched pattern name or empty string.
func (db *AISignatureDB) ScanForAPIKeys(content string) string {
db.mu.RLock()
defer db.mu.RUnlock()
for _, pattern := range db.apiKeyPatterns {
if pattern.Pattern.MatchString(content) {
return pattern.Name
}
}
return ""
}
// ServiceCount returns the number of known AI services.
func (db *AISignatureDB) ServiceCount() int {
db.mu.RLock()
defer db.mu.RUnlock()
return len(db.services)
}
// DomainPatternCount returns the number of compiled domain patterns.
func (db *AISignatureDB) DomainPatternCount() int {
db.mu.RLock()
defer db.mu.RUnlock()
return len(db.domainPatterns)
}
// AddService adds a custom AI service to the database.
func (db *AISignatureDB) AddService(svc AIServiceInfo) {
db.mu.Lock()
defer db.mu.Unlock()
db.services = append(db.services, svc)
for _, d := range svc.Domains {
pattern := domainToRegex(d)
db.domainPatterns = append(db.domainPatterns, &domainPattern{
original: d,
regex: pattern,
service: svc.Name,
})
}
}
// --- Network Detector ---
// NetworkEvent represents a network connection event for analysis.
type NetworkEvent struct {
User string `json:"user"`
Hostname string `json:"hostname"`
Destination string `json:"destination"` // Domain or IP
Port int `json:"port"`
HTTPHeaders map[string]string `json:"http_headers,omitempty"`
TLSJA3 string `json:"tls_ja3,omitempty"`
DataSize int64 `json:"data_size"`
Timestamp time.Time `json:"timestamp"`
}
// NetworkDetector analyzes network events for AI service access.
type NetworkDetector struct {
signatures *AISignatureDB
logger *slog.Logger
}
// NewNetworkDetector creates a new network detector with the default signature DB.
func NewNetworkDetector() *NetworkDetector {
return &NetworkDetector{
signatures: NewAISignatureDB(),
logger: slog.Default().With("component", "shadow-ai-network"),
}
}
// NewNetworkDetectorWithDB creates a detector with a custom signature database.
func NewNetworkDetectorWithDB(db *AISignatureDB) *NetworkDetector {
return &NetworkDetector{
signatures: db,
logger: slog.Default().With("component", "shadow-ai-network"),
}
}
// Analyze checks a network event for AI service access.
// Returns a ShadowAIEvent if detected, nil otherwise.
func (nd *NetworkDetector) Analyze(event NetworkEvent) *ShadowAIEvent {
// Check domain match.
if service := nd.signatures.MatchDomain(event.Destination); service != "" {
nd.logger.Info("AI domain detected",
"user", event.User,
"destination", event.Destination,
"service", service,
)
return &ShadowAIEvent{
UserID: event.User,
Hostname: event.Hostname,
Destination: event.Destination,
AIService: service,
DetectionMethod: DetectNetwork,
Action: "detected",
DataSize: event.DataSize,
Timestamp: event.Timestamp,
}
}
// Check HTTP header signatures.
if sig := nd.signatures.MatchHTTPHeaders(event.HTTPHeaders); sig != "" {
nd.logger.Info("AI HTTP signature detected",
"user", event.User,
"destination", event.Destination,
"signature", sig,
)
return &ShadowAIEvent{
UserID: event.User,
Hostname: event.Hostname,
Destination: event.Destination,
AIService: "unknown",
DetectionMethod: DetectHTTP,
Action: "detected",
DataSize: event.DataSize,
Timestamp: event.Timestamp,
Metadata: map[string]string{"http_signature": sig},
}
}
return nil
}
// SignatureDB returns the underlying signature database for extension.
func (nd *NetworkDetector) SignatureDB() *AISignatureDB {
return nd.signatures
}
// --- Behavioral Detector ---
// UserBehaviorProfile tracks a user's AI access behavior for anomaly detection.
type UserBehaviorProfile struct {
UserID string `json:"user_id"`
AccessFrequency float64 `json:"access_frequency"` // Requests per hour
DataVolumePerHour float64 `json:"data_volume_per_hour"` // Bytes per hour
KnownDestinations []string `json:"known_destinations"`
UpdatedAt time.Time `json:"updated_at"`
}
// BehavioralAlert is emitted when anomalous AI access is detected.
type BehavioralAlert struct {
UserID string `json:"user_id"`
AnomalyType string `json:"anomaly_type"` // "access_spike", "new_destination", "data_volume_spike"
Current float64 `json:"current"`
Baseline float64 `json:"baseline"`
ZScore float64 `json:"z_score"`
Destination string `json:"destination,omitempty"`
Severity string `json:"severity"`
}
// BehavioralDetector detects anomalous AI usage patterns per user.
type BehavioralDetector struct {
mu sync.RWMutex
baselines map[string]*UserBehaviorProfile
current map[string]*UserBehaviorProfile
alertBus chan BehavioralAlert
logger *slog.Logger
}
// NewBehavioralDetector creates a behavioral detector with a buffered alert bus.
func NewBehavioralDetector(alertBufSize int) *BehavioralDetector {
if alertBufSize <= 0 {
alertBufSize = 100
}
return &BehavioralDetector{
baselines: make(map[string]*UserBehaviorProfile),
current: make(map[string]*UserBehaviorProfile),
alertBus: make(chan BehavioralAlert, alertBufSize),
logger: slog.Default().With("component", "shadow-ai-behavioral"),
}
}
// RecordAccess records a single AI access attempt for behavioral tracking.
func (bd *BehavioralDetector) RecordAccess(userID, destination string, dataSize int64) {
bd.mu.Lock()
defer bd.mu.Unlock()
profile, ok := bd.current[userID]
if !ok {
profile = &UserBehaviorProfile{
UserID: userID,
}
bd.current[userID] = profile
}
profile.AccessFrequency++
profile.DataVolumePerHour += float64(dataSize)
profile.UpdatedAt = time.Now()
// Track destinations.
found := false
for _, d := range profile.KnownDestinations {
if d == destination {
found = true
break
}
}
if !found {
profile.KnownDestinations = append(profile.KnownDestinations, destination)
}
}
// SetBaseline sets the known baseline behavior for a user.
func (bd *BehavioralDetector) SetBaseline(userID string, profile *UserBehaviorProfile) {
bd.mu.Lock()
defer bd.mu.Unlock()
bd.baselines[userID] = profile
}
// DetectAnomalies compares current behavior to baselines and emits alerts.
func (bd *BehavioralDetector) DetectAnomalies() []BehavioralAlert {
bd.mu.RLock()
defer bd.mu.RUnlock()
var alerts []BehavioralAlert
for userID, current := range bd.current {
baseline, ok := bd.baselines[userID]
if !ok {
// No baseline — any AI access from this user is suspicious.
if current.AccessFrequency > 0 {
alert := BehavioralAlert{
UserID: userID,
AnomalyType: "first_ai_access",
Current: current.AccessFrequency,
Baseline: 0,
Severity: "WARNING",
}
alerts = append(alerts, alert)
bd.emitAlert(alert)
}
continue
}
// Z-score for access frequency.
if baseline.AccessFrequency > 0 {
zscore := (current.AccessFrequency - baseline.AccessFrequency) / math.Max(baseline.AccessFrequency*0.3, 1)
if math.Abs(zscore) > 3.0 {
alert := BehavioralAlert{
UserID: userID,
AnomalyType: "access_spike",
Current: current.AccessFrequency,
Baseline: baseline.AccessFrequency,
ZScore: zscore,
Severity: "WARNING",
}
alerts = append(alerts, alert)
bd.emitAlert(alert)
}
}
// Detect new AI destinations.
for _, dest := range current.KnownDestinations {
isNew := true
for _, known := range baseline.KnownDestinations {
if dest == known {
isNew = false
break
}
}
if isNew {
alert := BehavioralAlert{
UserID: userID,
AnomalyType: "new_ai_destination",
Destination: dest,
Severity: "HIGH",
}
alerts = append(alerts, alert)
bd.emitAlert(alert)
}
}
// Z-score for data volume.
if baseline.DataVolumePerHour > 0 {
zscore := (current.DataVolumePerHour - baseline.DataVolumePerHour) / math.Max(baseline.DataVolumePerHour*0.3, 1)
if math.Abs(zscore) > 3.0 {
alert := BehavioralAlert{
UserID: userID,
AnomalyType: "data_volume_spike",
Current: current.DataVolumePerHour,
Baseline: baseline.DataVolumePerHour,
ZScore: zscore,
Severity: "CRITICAL",
}
alerts = append(alerts, alert)
bd.emitAlert(alert)
}
}
}
return alerts
}
// Alerts returns the alert channel for consuming behavioral alerts.
func (bd *BehavioralDetector) Alerts() <-chan BehavioralAlert {
return bd.alertBus
}
// ResetCurrent clears the current period data (call after each analysis window).
func (bd *BehavioralDetector) ResetCurrent() {
bd.mu.Lock()
defer bd.mu.Unlock()
bd.current = make(map[string]*UserBehaviorProfile)
}
func (bd *BehavioralDetector) emitAlert(alert BehavioralAlert) {
select {
case bd.alertBus <- alert:
default:
bd.logger.Warn("behavioral alert bus full, dropping alert",
"user", alert.UserID,
"type", alert.AnomalyType,
)
}
}
// --- Default Data ---
func defaultAIServices() []AIServiceInfo {
return []AIServiceInfo{
{Name: "ChatGPT", Vendor: "OpenAI", Domains: []string{"chat.openai.com", "api.openai.com", "*.openai.com"}, Category: "llm"},
{Name: "Claude", Vendor: "Anthropic", Domains: []string{"claude.ai", "api.anthropic.com", "*.anthropic.com"}, Category: "llm"},
{Name: "Gemini", Vendor: "Google", Domains: []string{"gemini.google.com", "generativelanguage.googleapis.com", "aistudio.google.com"}, Category: "llm"},
{Name: "Copilot", Vendor: "Microsoft", Domains: []string{"copilot.microsoft.com", "*.copilot.microsoft.com"}, Category: "code_assist"},
{Name: "Cohere", Vendor: "Cohere", Domains: []string{"api.cohere.ai", "dashboard.cohere.com", "*.cohere.ai"}, Category: "llm"},
{Name: "AI21", Vendor: "AI21 Labs", Domains: []string{"api.ai21.com", "studio.ai21.com", "*.ai21.com"}, Category: "llm"},
{Name: "HuggingFace", Vendor: "Hugging Face", Domains: []string{"api-inference.huggingface.co", "huggingface.co", "*.huggingface.co"}, Category: "llm"},
{Name: "Replicate", Vendor: "Replicate", Domains: []string{"api.replicate.com", "replicate.com", "*.replicate.com"}, Category: "llm"},
{Name: "Mistral", Vendor: "Mistral AI", Domains: []string{"api.mistral.ai", "chat.mistral.ai", "*.mistral.ai"}, Category: "llm"},
{Name: "Perplexity", Vendor: "Perplexity", Domains: []string{"api.perplexity.ai", "perplexity.ai", "*.perplexity.ai"}, Category: "llm"},
{Name: "Groq", Vendor: "Groq", Domains: []string{"api.groq.com", "groq.com", "*.groq.com"}, Category: "llm"},
{Name: "Together", Vendor: "Together AI", Domains: []string{"api.together.xyz", "together.ai", "*.together.ai"}, Category: "llm"},
{Name: "Stability", Vendor: "Stability AI", Domains: []string{"api.stability.ai", "*.stability.ai"}, Category: "image_gen"},
{Name: "Midjourney", Vendor: "Midjourney", Domains: []string{"midjourney.com", "*.midjourney.com"}, Category: "image_gen"},
{Name: "DALL-E", Vendor: "OpenAI", Domains: []string{"labs.openai.com"}, Category: "image_gen"},
{Name: "Cursor", Vendor: "Cursor", Domains: []string{"api2.cursor.sh", "*.cursor.sh"}, Category: "code_assist"},
{Name: "Replit AI", Vendor: "Replit", Domains: []string{"replit.com", "*.replit.com"}, Category: "code_assist"},
{Name: "Codeium", Vendor: "Codeium", Domains: []string{"*.codeium.com", "codeium.com"}, Category: "code_assist"},
{Name: "Tabnine", Vendor: "Tabnine", Domains: []string{"*.tabnine.com", "tabnine.com"}, Category: "code_assist"},
{Name: "Qwen", Vendor: "Alibaba", Domains: []string{"dashscope.aliyuncs.com", "*.dashscope.aliyuncs.com"}, Category: "llm"},
{Name: "DeepSeek", Vendor: "DeepSeek", Domains: []string{"api.deepseek.com", "chat.deepseek.com", "*.deepseek.com"}, Category: "llm"},
{Name: "Kimi", Vendor: "Moonshot AI", Domains: []string{"api.moonshot.cn", "kimi.moonshot.cn", "*.moonshot.cn"}, Category: "llm"},
{Name: "Baidu ERNIE", Vendor: "Baidu", Domains: []string{"aip.baidubce.com", "erniebot.baidu.com"}, Category: "llm"},
{Name: "Jasper", Vendor: "Jasper", Domains: []string{"app.jasper.ai", "api.jasper.ai", "*.jasper.ai"}, Category: "llm"},
{Name: "Writer", Vendor: "Writer", Domains: []string{"writer.com", "api.writer.com", "*.writer.com"}, Category: "llm"},
{Name: "Notion AI", Vendor: "Notion", Domains: []string{"www.notion.so"}, Category: "productivity"},
{Name: "Grammarly AI", Vendor: "Grammarly", Domains: []string{"*.grammarly.com"}, Category: "productivity"},
{Name: "Runway", Vendor: "Runway", Domains: []string{"app.runwayml.com", "api.runwayml.com", "*.runwayml.com"}, Category: "video_gen"},
{Name: "Pika", Vendor: "Pika", Domains: []string{"pika.art", "*.pika.art"}, Category: "video_gen"},
{Name: "ElevenLabs", Vendor: "ElevenLabs", Domains: []string{"api.elevenlabs.io", "elevenlabs.io", "*.elevenlabs.io"}, Category: "audio_gen"},
{Name: "Suno", Vendor: "Suno", Domains: []string{"suno.com", "*.suno.com"}, Category: "audio_gen"},
{Name: "OpenRouter", Vendor: "OpenRouter", Domains: []string{"openrouter.ai", "*.openrouter.ai"}, Category: "llm"},
{Name: "Scale AI", Vendor: "Scale", Domains: []string{"scale.com", "api.scale.com", "*.scale.com"}, Category: "llm"},
{Name: "Inflection Pi", Vendor: "Inflection", Domains: []string{"pi.ai", "api.inflection.ai"}, Category: "llm"},
{Name: "Grok", Vendor: "xAI", Domains: []string{"grok.x.ai", "api.x.ai"}, Category: "llm"},
{Name: "Character.AI", Vendor: "Character.AI", Domains: []string{"character.ai", "*.character.ai"}, Category: "llm"},
{Name: "Poe", Vendor: "Quora", Domains: []string{"poe.com", "*.poe.com"}, Category: "llm"},
{Name: "You.com", Vendor: "You.com", Domains: []string{"you.com", "api.you.com"}, Category: "llm"},
{Name: "Phind", Vendor: "Phind", Domains: []string{"phind.com", "*.phind.com"}, Category: "llm"},
}
}
func defaultAPIKeyPatterns() []*APIKeyPattern {
return []*APIKeyPattern{
{Name: "OpenAI API Key", Pattern: regexp.MustCompile(`sk-[a-zA-Z0-9]{20,}T3BlbkFJ[a-zA-Z0-9]{20,}`), Entropy: 4.5},
{Name: "OpenAI Project Key", Pattern: regexp.MustCompile(`sk-proj-[a-zA-Z0-9\-_]{48,}`), Entropy: 4.5},
{Name: "Anthropic API Key", Pattern: regexp.MustCompile(`sk-ant-[a-zA-Z0-9\-_]{90,}`), Entropy: 4.5},
{Name: "Google AI API Key", Pattern: regexp.MustCompile(`AIza[0-9A-Za-z\-_]{35}`), Entropy: 4.0},
{Name: "HuggingFace Token", Pattern: regexp.MustCompile(`hf_[a-zA-Z0-9]{34}`), Entropy: 4.5},
{Name: "Groq API Key", Pattern: regexp.MustCompile(`gsk_[a-zA-Z0-9]{52}`), Entropy: 4.5},
{Name: "Cohere API Key", Pattern: regexp.MustCompile(`[a-zA-Z0-9]{10,}-[a-zA-Z0-9]{4,}-[a-zA-Z0-9]{4,}-[a-zA-Z0-9]{4,}-[a-zA-Z0-9]{12,}`), Entropy: 4.5},
{Name: "Replicate API Token", Pattern: regexp.MustCompile(`r8_[a-zA-Z0-9]{37}`), Entropy: 4.5},
}
}
// ServicesByCategory returns AI services grouped by category.
func ServicesByCategory() map[string][]AIServiceInfo {
services := defaultAIServices()
result := make(map[string][]AIServiceInfo)
for _, svc := range services {
result[svc.Category] = append(result[svc.Category], svc)
}
// Sort each category by name for deterministic output.
for cat := range result {
sort.Slice(result[cat], func(i, j int) bool {
return result[cat][i].Name < result[cat][j].Name
})
}
return result
}

View file

@ -0,0 +1,353 @@
package shadow_ai
import (
"crypto/sha256"
"fmt"
"regexp"
"strings"
"sync"
"time"
)
// --- Document Review Bridge ---
// Controlled gateway for AI access: scans documents for secrets and PII,
// supports content redaction, and routes through the approval workflow.
// DocReviewStatus tracks the lifecycle of a document review.
type DocReviewStatus string
const (
DocReviewPending DocReviewStatus = "pending"
DocReviewScanning DocReviewStatus = "scanning"
DocReviewClean DocReviewStatus = "clean"
DocReviewRedacted DocReviewStatus = "redacted"
DocReviewBlocked DocReviewStatus = "blocked"
DocReviewApproved DocReviewStatus = "approved"
)
// ScanResult contains the results of scanning a document.
type ScanResult struct {
DocumentID string `json:"document_id"`
Status DocReviewStatus `json:"status"`
PIIFound []PIIMatch `json:"pii_found,omitempty"`
SecretsFound []SecretMatch `json:"secrets_found,omitempty"`
DataClass DataClassification `json:"data_classification"`
ContentHash string `json:"content_hash"`
ScannedAt time.Time `json:"scanned_at"`
SizeBytes int `json:"size_bytes"`
}
// PIIMatch represents a detected PII pattern in content.
type PIIMatch struct {
Type string `json:"type"` // "email", "phone", "ssn", "credit_card", "passport"
Location int `json:"location"` // Character offset
Length int `json:"length"`
Masked string `json:"masked"` // Redacted value, e.g., "j***@example.com"
}
// SecretMatch represents a detected secret/API key in content.
type SecretMatch struct {
Type string `json:"type"` // "api_key", "password", "token", "private_key"
Location int `json:"location"`
Length int `json:"length"`
Provider string `json:"provider"` // "OpenAI", "AWS", "GitHub", etc.
}
// DocBridge manages document scanning, redaction, and review workflow.
type DocBridge struct {
mu sync.RWMutex
reviews map[string]*ScanResult
piiPatterns []*piiPattern
secretPats []secretPattern // Cached compiled patterns
signatures *AISignatureDB // Reused across scans
maxDocSize int // bytes
}
type piiPattern struct {
name string
regex *regexp.Regexp
maskFn func(string) string
}
// NewDocBridge creates a new Document Review Bridge.
func NewDocBridge() *DocBridge {
return &DocBridge{
reviews: make(map[string]*ScanResult),
piiPatterns: defaultPIIPatterns(),
secretPats: secretPatterns(),
signatures: NewAISignatureDB(),
maxDocSize: 10 * 1024 * 1024, // 10 MB
}
}
// ScanDocument scans content for PII and secrets, classifies data, returns result.
func (db *DocBridge) ScanDocument(docID, content, userID string) *ScanResult {
result := &ScanResult{
DocumentID: docID,
Status: DocReviewScanning,
ScannedAt: time.Now(),
SizeBytes: len(content),
}
// Content hash for dedup.
h := sha256.Sum256([]byte(content))
result.ContentHash = fmt.Sprintf("%x", h[:])
// Size check.
if len(content) > db.maxDocSize {
result.Status = DocReviewBlocked
result.DataClass = DataCritical
db.store(result)
return result
}
// Scan for PII.
result.PIIFound = db.scanPII(content)
// Scan for secrets (reuse cached signature DB).
if keyType := db.signatures.ScanForAPIKeys(content); keyType != "" {
result.SecretsFound = append(result.SecretsFound, SecretMatch{
Type: "api_key",
Provider: keyType,
})
}
// Scan for additional secret patterns.
result.SecretsFound = append(result.SecretsFound, db.scanSecrets(content)...)
// Classify data based on findings.
result.DataClass = db.classifyData(result)
// Set status based on findings.
if len(result.SecretsFound) > 0 {
result.Status = DocReviewBlocked
} else if len(result.PIIFound) > 0 {
result.Status = DocReviewRedacted
} else {
result.Status = DocReviewClean
}
db.store(result)
return result
}
// RedactContent replaces PII and secrets in content with masked values.
func (db *DocBridge) RedactContent(content string) string {
for _, p := range db.piiPatterns {
content = p.regex.ReplaceAllStringFunc(content, p.maskFn)
}
// Redact common secret patterns (cached).
for _, sp := range db.secretPats {
content = sp.regex.ReplaceAllString(content, sp.replacement)
}
return content
}
// GetReview returns a scan result by document ID.
func (db *DocBridge) GetReview(docID string) (*ScanResult, bool) {
db.mu.RLock()
defer db.mu.RUnlock()
r, ok := db.reviews[docID]
if !ok {
return nil, false
}
cp := *r
return &cp, true
}
// RecentReviews returns the N most recent reviews.
func (db *DocBridge) RecentReviews(limit int) []ScanResult {
db.mu.RLock()
defer db.mu.RUnlock()
results := make([]ScanResult, 0, len(db.reviews))
for _, r := range db.reviews {
results = append(results, *r)
}
// Sort by time desc (simple bubble for bounded set).
for i := 0; i < len(results); i++ {
for j := i + 1; j < len(results); j++ {
if results[j].ScannedAt.After(results[i].ScannedAt) {
results[i], results[j] = results[j], results[i]
}
}
}
if len(results) > limit {
results = results[:limit]
}
return results
}
// Stats returns aggregate document review statistics.
func (db *DocBridge) Stats() map[string]int {
db.mu.RLock()
defer db.mu.RUnlock()
stats := map[string]int{
"total": len(db.reviews),
"clean": 0,
"redacted": 0,
"blocked": 0,
}
for _, r := range db.reviews {
switch r.Status {
case DocReviewClean:
stats["clean"]++
case DocReviewRedacted:
stats["redacted"]++
case DocReviewBlocked:
stats["blocked"]++
}
}
return stats
}
func (db *DocBridge) store(result *ScanResult) {
db.mu.Lock()
defer db.mu.Unlock()
db.reviews[result.DocumentID] = result
}
// scanPII runs all PII patterns against content.
func (db *DocBridge) scanPII(content string) []PIIMatch {
var matches []PIIMatch
for _, p := range db.piiPatterns {
locs := p.regex.FindAllStringIndex(content, -1)
for _, loc := range locs {
matched := content[loc[0]:loc[1]]
matches = append(matches, PIIMatch{
Type: p.name,
Location: loc[0],
Length: loc[1] - loc[0],
Masked: p.maskFn(matched),
})
}
}
return matches
}
// scanSecrets scans for common secret patterns beyond AI API keys.
func (db *DocBridge) scanSecrets(content string) []SecretMatch {
var matches []SecretMatch
for _, sp := range db.secretPats {
locs := sp.regex.FindAllStringIndex(content, -1)
for _, loc := range locs {
matches = append(matches, SecretMatch{
Type: sp.secretType,
Location: loc[0],
Length: loc[1] - loc[0],
Provider: sp.provider,
})
}
}
return matches
}
// classifyData determines the data classification level based on scan results.
func (db *DocBridge) classifyData(result *ScanResult) DataClassification {
if len(result.SecretsFound) > 0 {
return DataCritical
}
hasSensitivePII := false
for _, pii := range result.PIIFound {
switch pii.Type {
case "ssn", "credit_card", "passport":
return DataCritical
case "email", "phone":
hasSensitivePII = true
}
}
if hasSensitivePII {
return DataConfidential
}
if result.SizeBytes > 1024*1024 { // >1MB
return DataInternal
}
return DataPublic
}
// --- PII Patterns ---
func defaultPIIPatterns() []*piiPattern {
return []*piiPattern{
{
name: "email",
regex: regexp.MustCompile(`[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}`),
maskFn: func(s string) string {
parts := strings.SplitN(s, "@", 2)
if len(parts) != 2 {
return "***@***"
}
if len(parts[0]) <= 1 {
return "*@" + parts[1]
}
return string(parts[0][0]) + "***@" + parts[1]
},
},
{
name: "phone",
regex: regexp.MustCompile(`\+?[1-9]\d{0,2}[\s\-]?\(?\d{3}\)?[\s\-]?\d{3}[\s\-]?\d{2,4}`),
maskFn: func(s string) string {
if len(s) < 4 {
return "***"
}
return s[:2] + strings.Repeat("*", len(s)-4) + s[len(s)-2:]
},
},
{
name: "ssn",
regex: regexp.MustCompile(`\b\d{3}-\d{2}-\d{4}\b`),
maskFn: func(_ string) string {
return "***-**-****"
},
},
{
name: "credit_card",
regex: regexp.MustCompile(`\b(?:\d{4}[\s\-]?){3}\d{4}\b`),
maskFn: func(s string) string {
clean := strings.ReplaceAll(strings.ReplaceAll(s, "-", ""), " ", "")
if len(clean) < 4 {
return "****"
}
return strings.Repeat("*", len(clean)-4) + clean[len(clean)-4:]
},
},
{
name: "passport",
regex: regexp.MustCompile(`\b[A-Z]{1,2}\d{6,9}\b`),
maskFn: func(s string) string {
if len(s) <= 2 {
return "**"
}
return s[:2] + strings.Repeat("*", len(s)-2)
},
},
}
}
type secretPattern struct {
secretType string
provider string
regex *regexp.Regexp
replacement string
}
func secretPatterns() []secretPattern {
return []secretPattern{
{secretType: "aws_key", provider: "AWS", regex: regexp.MustCompile(`AKIA[0-9A-Z]{16}`), replacement: "[AWS_KEY_REDACTED]"},
{secretType: "github_token", provider: "GitHub", regex: regexp.MustCompile(`ghp_[a-zA-Z0-9]{36}`), replacement: "[GITHUB_TOKEN_REDACTED]"},
{secretType: "github_token", provider: "GitHub", regex: regexp.MustCompile(`github_pat_[a-zA-Z0-9_]{82}`), replacement: "[GITHUB_PAT_REDACTED]"},
{secretType: "slack_token", provider: "Slack", regex: regexp.MustCompile(`xoxb-[0-9]{10,13}-[0-9]{10,13}-[a-zA-Z0-9]{24}`), replacement: "[SLACK_TOKEN_REDACTED]"},
{secretType: "private_key", provider: "Generic", regex: regexp.MustCompile(`-----BEGIN (?:RSA |EC |DSA )?PRIVATE KEY-----`), replacement: "[PRIVATE_KEY_REDACTED]"},
{secretType: "password", provider: "Generic", regex: regexp.MustCompile(`(?i)password\s*[=:]\s*['"]?[^\s'"]{8,}`), replacement: "[PASSWORD_REDACTED]"},
{secretType: "connection_string", provider: "Database", regex: regexp.MustCompile(`(?i)(?:mysql|postgres|mongodb)://[^\s]+`), replacement: "[DB_CONN_REDACTED]"},
}
}

View file

@ -0,0 +1,148 @@
package shadow_ai
import (
"context"
"fmt"
"log/slog"
"time"
)
// FallbackManager provides priority-based enforcement with graceful degradation.
// Tries enforcement points in priority order; falls back to detect_only if all are offline.
type FallbackManager struct {
registry *PluginRegistry
priority []PluginType // e.g., ["proxy", "firewall", "edr"]
strategy string // "detect_only" | "alert_only"
logger *slog.Logger
// Event logging for detect-only fallback.
eventLogFn func(event ShadowAIEvent)
}
// NewFallbackManager creates a new fallback manager with the given enforcement priority.
func NewFallbackManager(registry *PluginRegistry, strategy string) *FallbackManager {
if strategy == "" {
strategy = "detect_only"
}
return &FallbackManager{
registry: registry,
priority: []PluginType{PluginTypeProxy, PluginTypeFirewall, PluginTypeEDR},
strategy: strategy,
logger: slog.Default().With("component", "shadow-ai-fallback"),
}
}
// SetEventLogger sets the callback for logging detection-only events.
func (fm *FallbackManager) SetEventLogger(fn func(ShadowAIEvent)) {
fm.eventLogFn = fn
}
// BlockDomain attempts to block a domain using the highest-priority healthy plugin.
// Returns the vendor that enforced, or falls back to detect_only mode.
func (fm *FallbackManager) BlockDomain(ctx context.Context, domain, reason string) (enforcedBy string, err error) {
for _, pType := range fm.priority {
plugins := fm.registry.GetByType(pType)
for _, plugin := range plugins {
ne, ok := plugin.(NetworkEnforcer)
if !ok {
// Try WebGateway for URL-based blocking.
if wg, ok := plugin.(WebGateway); ok {
vendor := wg.Vendor()
if !fm.registry.IsHealthy(vendor) {
continue
}
if err := wg.BlockURL(ctx, domain, reason); err != nil {
fm.logger.Warn("block failed on gateway", "vendor", vendor, "error", err)
continue
}
return vendor, nil
}
continue
}
vendor := ne.Vendor()
if !fm.registry.IsHealthy(vendor) {
continue
}
if err := ne.BlockDomain(ctx, domain, reason); err != nil {
fm.logger.Warn("block failed on enforcer", "vendor", vendor, "error", err)
continue
}
return vendor, nil
}
}
// All enforcement points unavailable — fallback.
fm.logger.Warn("all enforcement points unavailable, falling to detect_only",
"domain", domain,
"strategy", fm.strategy,
)
fm.logDetectOnly(domain, reason)
return "", nil
}
// BlockIP attempts to block an IP using the highest-priority healthy firewall.
func (fm *FallbackManager) BlockIP(ctx context.Context, ip string, duration time.Duration, reason string) (enforcedBy string, err error) {
enforcers := fm.registry.GetNetworkEnforcers()
for _, ne := range enforcers {
vendor := ne.Vendor()
if !fm.registry.IsHealthy(vendor) {
continue
}
if err := ne.BlockIP(ctx, ip, duration, reason); err != nil {
fm.logger.Warn("block IP failed", "vendor", vendor, "error", err)
continue
}
return vendor, nil
}
fm.logger.Warn("no healthy enforcer for IP block, falling to detect_only",
"ip", ip,
"strategy", fm.strategy,
)
fm.logDetectOnly(ip, reason)
return "", nil
}
// IsolateHost attempts to isolate a host using the highest-priority healthy EDR.
func (fm *FallbackManager) IsolateHost(ctx context.Context, hostname string) (enforcedBy string, err error) {
controllers := fm.registry.GetEndpointControllers()
for _, ec := range controllers {
vendor := ec.Vendor()
if !fm.registry.IsHealthy(vendor) {
continue
}
if err := ec.IsolateHost(ctx, hostname); err != nil {
fm.logger.Warn("isolate failed", "vendor", vendor, "error", err)
continue
}
return vendor, nil
}
fm.logger.Warn("no healthy EDR for host isolation, falling to detect_only",
"hostname", hostname,
"strategy", fm.strategy,
)
return "", fmt.Errorf("no healthy EDR available for host isolation")
}
// logDetectOnly records a detection-only event when no enforcement is possible.
func (fm *FallbackManager) logDetectOnly(target, reason string) {
if fm.eventLogFn != nil {
fm.eventLogFn(ShadowAIEvent{
Destination: target,
DetectionMethod: DetectNetwork,
Action: "detect_only",
Metadata: map[string]string{
"reason": reason,
"fallback_strategy": fm.strategy,
},
Timestamp: time.Now(),
})
}
}
// Strategy returns the configured fallback strategy.
func (fm *FallbackManager) Strategy() string {
return fm.strategy
}

View file

@ -0,0 +1,163 @@
package shadow_ai
import (
"context"
"fmt"
"log/slog"
"sync"
"time"
)
// PluginStatus represents a plugin's operational state.
type PluginStatus string
const (
PluginStatusHealthy PluginStatus = "healthy"
PluginStatusDegraded PluginStatus = "degraded"
PluginStatusOffline PluginStatus = "offline"
)
// PluginHealth tracks the health state of a single plugin.
type PluginHealth struct {
Vendor string `json:"vendor"`
Type PluginType `json:"type"`
Status PluginStatus `json:"status"`
LastCheck time.Time `json:"last_check"`
Consecutive int `json:"consecutive_failures"`
Latency time.Duration `json:"latency"`
LastError string `json:"last_error,omitempty"`
}
// MaxConsecutivePluginFailures before marking offline.
const MaxConsecutivePluginFailures = 3
// HealthChecker performs continuous health monitoring of all registered plugins.
type HealthChecker struct {
mu sync.RWMutex
registry *PluginRegistry
interval time.Duration
alertFn func(vendor string, status PluginStatus, msg string)
logger *slog.Logger
}
// NewHealthChecker creates a health checker that monitors plugin health.
func NewHealthChecker(registry *PluginRegistry, interval time.Duration, alertFn func(string, PluginStatus, string)) *HealthChecker {
if interval <= 0 {
interval = 30 * time.Second
}
return &HealthChecker{
registry: registry,
interval: interval,
alertFn: alertFn,
logger: slog.Default().With("component", "shadow-ai-health"),
}
}
// Start begins continuous health monitoring. Blocks until ctx is cancelled.
func (hc *HealthChecker) Start(ctx context.Context) {
hc.logger.Info("health checker started", "interval", hc.interval)
ticker := time.NewTicker(hc.interval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
hc.logger.Info("health checker stopped")
return
case <-ticker.C:
hc.checkAllPlugins(ctx)
}
}
}
// checkAllPlugins runs health checks on all registered plugins.
func (hc *HealthChecker) checkAllPlugins(ctx context.Context) {
vendors := hc.registry.Vendors()
for _, vendor := range vendors {
plugin, ok := hc.registry.Get(vendor)
if !ok {
continue
}
existing, _ := hc.registry.GetHealth(vendor)
if existing == nil {
continue
}
start := time.Now()
err := hc.checkPlugin(ctx, plugin)
latency := time.Since(start)
health := &PluginHealth{
Vendor: vendor,
Type: existing.Type,
LastCheck: time.Now(),
Latency: latency,
}
if err != nil {
health.Consecutive = existing.Consecutive + 1
health.LastError = err.Error()
if health.Consecutive >= MaxConsecutivePluginFailures {
health.Status = PluginStatusOffline
if existing.Status != PluginStatusOffline {
hc.logger.Error("plugin went OFFLINE",
"vendor", vendor,
"consecutive", health.Consecutive,
"error", err,
)
if hc.alertFn != nil {
hc.alertFn(vendor, PluginStatusOffline,
fmt.Sprintf("Plugin %s offline after %d consecutive failures: %v",
vendor, health.Consecutive, err))
}
}
} else {
health.Status = PluginStatusDegraded
hc.logger.Warn("plugin health check failed",
"vendor", vendor,
"consecutive", health.Consecutive,
"error", err,
)
}
} else {
health.Status = PluginStatusHealthy
health.Consecutive = 0
// Log recovery if previously degraded/offline.
if existing.Status != PluginStatusHealthy {
hc.logger.Info("plugin recovered", "vendor", vendor, "latency", latency)
if hc.alertFn != nil {
hc.alertFn(vendor, PluginStatusHealthy,
fmt.Sprintf("Plugin %s recovered, latency %s", vendor, latency))
}
}
}
hc.registry.SetHealth(vendor, health)
}
}
// checkPlugin runs the health check for a single plugin.
func (hc *HealthChecker) checkPlugin(ctx context.Context, plugin interface{}) error {
checkCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
switch p := plugin.(type) {
case NetworkEnforcer:
return p.HealthCheck(checkCtx)
case EndpointController:
return p.HealthCheck(checkCtx)
case WebGateway:
return p.HealthCheck(checkCtx)
default:
return fmt.Errorf("plugin does not implement HealthCheck")
}
}
// CheckNow runs an immediate health check on all plugins (non-blocking).
func (hc *HealthChecker) CheckNow(ctx context.Context) {
hc.checkAllPlugins(ctx)
}

View file

@ -0,0 +1,225 @@
// Package shadow_ai implements the Sentinel Shadow AI Control Module.
//
// Five levels of shadow AI management:
//
// L1 — Universal Integration Layer: plugin-based enforcement (firewall, EDR, proxy)
// L2 — Detection Engine: network signatures, endpoint, API keys, behavioral
// L3 — Document Review Bridge: controlled LLM access with PII/secret scanning
// L4 — Approval Workflow: tiered data classification and manager/SOC approval
// L5 — SOC Integration: dashboard, correlation rules, playbooks, compliance
package shadow_ai
import (
"context"
"time"
)
// --- Plugin Interfaces ---
// NetworkEnforcer is the universal interface for ALL firewalls.
// Implementations: Check Point, Cisco ASA/FMC, Palo Alto, Fortinet.
type NetworkEnforcer interface {
// BlockIP blocks an IP address for the given duration.
BlockIP(ctx context.Context, ip string, duration time.Duration, reason string) error
// BlockDomain blocks a domain name.
BlockDomain(ctx context.Context, domain string, reason string) error
// UnblockIP removes an IP block.
UnblockIP(ctx context.Context, ip string) error
// UnblockDomain removes a domain block.
UnblockDomain(ctx context.Context, domain string) error
// HealthCheck verifies the firewall API is reachable.
HealthCheck(ctx context.Context) error
// Vendor returns the vendor identifier (e.g., "checkpoint", "cisco", "paloalto").
Vendor() string
}
// EndpointController is the universal interface for ALL EDR systems.
// Implementations: CrowdStrike, SentinelOne, Microsoft Defender.
type EndpointController interface {
// IsolateHost quarantines a host from the network.
IsolateHost(ctx context.Context, hostname string) error
// ReleaseHost removes host isolation.
ReleaseHost(ctx context.Context, hostname string) error
// KillProcess terminates a process on a remote host.
KillProcess(ctx context.Context, hostname string, pid int) error
// QuarantineFile moves a file to quarantine on a remote host.
QuarantineFile(ctx context.Context, hostname string, path string) error
// HealthCheck verifies the EDR API is reachable.
HealthCheck(ctx context.Context) error
// Vendor returns the vendor identifier (e.g., "crowdstrike", "sentinelone", "defender").
Vendor() string
}
// WebGateway is the universal interface for ALL proxy/CASB systems.
// Implementations: Zscaler, Netskope, Squid, BlueCoat.
type WebGateway interface {
// BlockURL adds a URL to the blocklist.
BlockURL(ctx context.Context, url string, reason string) error
// UnblockURL removes a URL from the blocklist.
UnblockURL(ctx context.Context, url string) error
// BlockCategory blocks an entire URL category (e.g., "Artificial Intelligence").
BlockCategory(ctx context.Context, category string) error
// HealthCheck verifies the gateway API is reachable.
HealthCheck(ctx context.Context) error
// Vendor returns the vendor identifier (e.g., "zscaler", "netskope", "squid").
Vendor() string
}
// Initializer is implemented by plugins that need configuration before use.
type Initializer interface {
Initialize(config map[string]interface{}) error
}
// --- Plugin Configuration ---
// PluginType categorizes enforcement points.
type PluginType string
const (
PluginTypeFirewall PluginType = "firewall"
PluginTypeEDR PluginType = "edr"
PluginTypeProxy PluginType = "proxy"
PluginTypeDNS PluginType = "dns"
)
// PluginConfig defines a vendor plugin configuration loaded from YAML.
type PluginConfig struct {
Type PluginType `yaml:"type" json:"type"`
Vendor string `yaml:"vendor" json:"vendor"`
Enabled bool `yaml:"enabled" json:"enabled"`
Config map[string]interface{} `yaml:"config" json:"config"`
}
// IntegrationConfig is the top-level Shadow AI configuration.
type IntegrationConfig struct {
Plugins []PluginConfig `yaml:"plugins" json:"plugins"`
FallbackStrategy string `yaml:"fallback_strategy" json:"fallback_strategy"` // "detect_only" | "alert_only"
HealthCheckInterval time.Duration `yaml:"health_check_interval" json:"health_check_interval"` // default: 30s
}
// --- Domain Types ---
// DetectionMethod identifies how a shadow AI usage was detected.
type DetectionMethod string
const (
DetectNetwork DetectionMethod = "network" // Domain/IP match
DetectHTTP DetectionMethod = "http" // HTTP header signature
DetectTLS DetectionMethod = "tls" // TLS/JA3 fingerprint
DetectProcess DetectionMethod = "process" // AI tool process execution
DetectAPIKey DetectionMethod = "api_key" // AI API key in payload
DetectBehavioral DetectionMethod = "behavioral" // Anomalous AI access pattern
DetectClipboard DetectionMethod = "clipboard" // Large clipboard → AI browser pattern
)
// DataClassification determines the approval tier required.
type DataClassification string
const (
DataPublic DataClassification = "PUBLIC"
DataInternal DataClassification = "INTERNAL"
DataConfidential DataClassification = "CONFIDENTIAL"
DataCritical DataClassification = "CRITICAL"
)
// ShadowAIEvent is a detected shadow AI usage attempt.
type ShadowAIEvent struct {
ID string `json:"id"`
UserID string `json:"user_id"`
Hostname string `json:"hostname"`
Destination string `json:"destination"` // Target AI service domain/IP
AIService string `json:"ai_service"` // "chatgpt", "claude", "gemini", etc.
DetectionMethod DetectionMethod `json:"detection_method"`
Action string `json:"action"` // "blocked", "allowed", "pending"
EnforcedBy string `json:"enforced_by"` // Plugin vendor that enforced
DataSize int64 `json:"data_size"` // Bytes sent to AI
Timestamp time.Time `json:"timestamp"`
Metadata map[string]string `json:"metadata,omitempty"`
}
// AIServiceInfo describes a known AI service for signature matching.
type AIServiceInfo struct {
Name string `json:"name"` // "ChatGPT", "Claude", "Gemini"
Vendor string `json:"vendor"` // "OpenAI", "Anthropic", "Google"
Domains []string `json:"domains"` // ["*.openai.com", "chat.openai.com"]
Category string `json:"category"` // "llm", "image_gen", "code_assist"
}
// BlockRequest is an API request to manually block a target.
type BlockRequest struct {
TargetType string `json:"target_type"` // "ip", "domain", "user"
Target string `json:"target"`
Duration time.Duration `json:"duration"`
Reason string `json:"reason"`
BlockedBy string `json:"blocked_by"` // RBAC user
}
// ShadowAIStats provides aggregate statistics for the dashboard.
type ShadowAIStats struct {
TimeRange string `json:"time_range"` // "24h", "7d", "30d"
Total int `json:"total_attempts"`
Blocked int `json:"blocked"`
Approved int `json:"approved"`
Pending int `json:"pending"`
ByService map[string]int `json:"by_service"`
ByDepartment map[string]int `json:"by_department"`
TopViolators []Violator `json:"top_violators"`
}
// Violator tracks a user's shadow AI violation count.
type Violator struct {
UserID string `json:"user_id"`
Attempts int `json:"attempts"`
}
// ApprovalTier defines the approval requirements for a data classification level.
type ApprovalTier struct {
Name string `yaml:"name" json:"name"`
DataClass DataClassification `yaml:"data_class" json:"data_class"`
ApprovalNeeded []string `yaml:"approval_needed" json:"approval_needed"` // ["manager"], ["manager", "soc"], ["ciso"]
SLA time.Duration `yaml:"sla" json:"sla"`
AutoApprove bool `yaml:"auto_approve" json:"auto_approve"`
}
// ApprovalRequest tracks a pending approval for AI access.
type ApprovalRequest struct {
ID string `json:"id"`
DocID string `json:"doc_id"`
UserID string `json:"user_id"`
Tier string `json:"tier"`
DataClass DataClassification `json:"data_class"`
Status string `json:"status"` // "pending", "approved", "denied", "expired"
ApprovedBy string `json:"approved_by,omitempty"`
DeniedBy string `json:"denied_by,omitempty"`
Reason string `json:"reason,omitempty"`
CreatedAt time.Time `json:"created_at"`
ExpiresAt time.Time `json:"expires_at"`
ResolvedAt time.Time `json:"resolved_at,omitempty"`
}
// ComplianceReport is the Shadow AI compliance report for GDPR/SOC2/EU AI Act.
type ComplianceReport struct {
GeneratedAt time.Time `json:"generated_at"`
Period string `json:"period"` // "monthly", "quarterly"
TotalInteractions int `json:"total_interactions"`
BlockedAttempts int `json:"blocked_attempts"`
ApprovedReviews int `json:"approved_reviews"`
PIIDetected int `json:"pii_detected"`
SecretsDetected int `json:"secrets_detected"`
AuditComplete bool `json:"audit_complete"`
Regulations []string `json:"regulations"` // ["GDPR", "SOC2", "EU AI Act"]
}

View file

@ -0,0 +1,212 @@
package shadow_ai
import (
"context"
"fmt"
"log/slog"
"time"
)
// --- Vendor Plugin Stubs ---
// Reference implementations for major security vendors.
// These stubs implement the full interface with logging but no real API calls.
// Production deployments replace these with real vendor SDK integrations.
// CheckPointEnforcer is a stub implementation for Check Point firewalls.
type CheckPointEnforcer struct {
apiURL string
apiKey string
logger *slog.Logger
}
func NewCheckPointEnforcer() *CheckPointEnforcer {
return &CheckPointEnforcer{
logger: slog.Default().With("component", "shadow-ai-plugin-checkpoint"),
}
}
func (c *CheckPointEnforcer) Initialize(config map[string]interface{}) error {
if url, ok := config["api_url"].(string); ok {
c.apiURL = url
}
if key, ok := config["api_key"].(string); ok {
c.apiKey = key
}
if c.apiURL == "" {
return fmt.Errorf("checkpoint: api_url required")
}
c.logger.Info("initialized", "api_url", c.apiURL)
return nil
}
func (c *CheckPointEnforcer) BlockIP(_ context.Context, ip string, duration time.Duration, reason string) error {
c.logger.Info("block IP", "ip", ip, "duration", duration, "reason", reason)
// Stub: would call Check Point Management API POST /web_api/add-host
return nil
}
func (c *CheckPointEnforcer) BlockDomain(_ context.Context, domain string, reason string) error {
c.logger.Info("block domain", "domain", domain, "reason", reason)
// Stub: would create application-site-category block rule
return nil
}
func (c *CheckPointEnforcer) UnblockIP(_ context.Context, ip string) error {
c.logger.Info("unblock IP", "ip", ip)
return nil
}
func (c *CheckPointEnforcer) UnblockDomain(_ context.Context, domain string) error {
c.logger.Info("unblock domain", "domain", domain)
return nil
}
func (c *CheckPointEnforcer) HealthCheck(ctx context.Context) error {
if c.apiURL == "" {
return fmt.Errorf("not configured")
}
// Stub: would call GET /web_api/show-session
return nil
}
func (c *CheckPointEnforcer) Vendor() string { return "checkpoint" }
// CrowdStrikeController is a stub implementation for CrowdStrike Falcon EDR.
type CrowdStrikeController struct {
clientID string
clientSecret string
baseURL string
logger *slog.Logger
}
func NewCrowdStrikeController() *CrowdStrikeController {
return &CrowdStrikeController{
baseURL: "https://api.crowdstrike.com",
logger: slog.Default().With("component", "shadow-ai-plugin-crowdstrike"),
}
}
func (cs *CrowdStrikeController) Initialize(config map[string]interface{}) error {
if id, ok := config["client_id"].(string); ok {
cs.clientID = id
}
if secret, ok := config["client_secret"].(string); ok {
cs.clientSecret = secret
}
if url, ok := config["base_url"].(string); ok {
cs.baseURL = url
}
if cs.clientID == "" {
return fmt.Errorf("crowdstrike: client_id required")
}
cs.logger.Info("initialized", "base_url", cs.baseURL)
return nil
}
func (cs *CrowdStrikeController) IsolateHost(_ context.Context, hostname string) error {
cs.logger.Info("isolate host", "hostname", hostname)
// Stub: would call POST /devices/entities/devices-actions/v2?action_name=contain
return nil
}
func (cs *CrowdStrikeController) ReleaseHost(_ context.Context, hostname string) error {
cs.logger.Info("release host", "hostname", hostname)
// Stub: would call POST /devices/entities/devices-actions/v2?action_name=lift_containment
return nil
}
func (cs *CrowdStrikeController) KillProcess(_ context.Context, hostname string, pid int) error {
cs.logger.Info("kill process", "hostname", hostname, "pid", pid)
// Stub: would use RTR session to kill process
return nil
}
func (cs *CrowdStrikeController) QuarantineFile(_ context.Context, hostname, path string) error {
cs.logger.Info("quarantine file", "hostname", hostname, "path", path)
return nil
}
func (cs *CrowdStrikeController) HealthCheck(ctx context.Context) error {
if cs.clientID == "" {
return fmt.Errorf("not configured")
}
// Stub: would call GET /sensors/queries/sensors/v1?limit=1
return nil
}
func (cs *CrowdStrikeController) Vendor() string { return "crowdstrike" }
// ZscalerGateway is a stub implementation for Zscaler Internet Access.
type ZscalerGateway struct {
cloudName string
apiKey string
username string
password string
logger *slog.Logger
}
func NewZscalerGateway() *ZscalerGateway {
return &ZscalerGateway{
logger: slog.Default().With("component", "shadow-ai-plugin-zscaler"),
}
}
func (z *ZscalerGateway) Initialize(config map[string]interface{}) error {
if cloud, ok := config["cloud_name"].(string); ok {
z.cloudName = cloud
}
if key, ok := config["api_key"].(string); ok {
z.apiKey = key
}
if user, ok := config["username"].(string); ok {
z.username = user
}
if pass, ok := config["password"].(string); ok {
z.password = pass
}
if z.cloudName == "" {
return fmt.Errorf("zscaler: cloud_name required")
}
z.logger.Info("initialized", "cloud", z.cloudName)
return nil
}
func (z *ZscalerGateway) BlockURL(_ context.Context, url, reason string) error {
z.logger.Info("block URL", "url", url, "reason", reason)
// Stub: would call PUT /webApplicationRules to add URL to block list
return nil
}
func (z *ZscalerGateway) UnblockURL(_ context.Context, url string) error {
z.logger.Info("unblock URL", "url", url)
return nil
}
func (z *ZscalerGateway) BlockCategory(_ context.Context, category string) error {
z.logger.Info("block category", "category", category)
// Stub: would update URL category policy to BLOCK
return nil
}
func (z *ZscalerGateway) HealthCheck(ctx context.Context) error {
if z.cloudName == "" {
return fmt.Errorf("not configured")
}
// Stub: would call GET /status
return nil
}
func (z *ZscalerGateway) Vendor() string { return "zscaler" }
// RegisterDefaultPlugins registers all built-in vendor plugin factories.
func RegisterDefaultPlugins(registry *PluginRegistry) {
registry.RegisterFactory(PluginTypeFirewall, "checkpoint", func() interface{} {
return NewCheckPointEnforcer()
})
registry.RegisterFactory(PluginTypeEDR, "crowdstrike", func() interface{} {
return NewCrowdStrikeController()
})
registry.RegisterFactory(PluginTypeProxy, "zscaler", func() interface{} {
return NewZscalerGateway()
})
}

View file

@ -0,0 +1,212 @@
package shadow_ai
import (
"fmt"
"log/slog"
"sync"
)
// PluginFactory creates a new plugin instance.
type PluginFactory func() interface{}
// PluginRegistry manages vendor plugin registration, loading, and lifecycle.
// Thread-safe via sync.RWMutex.
type PluginRegistry struct {
mu sync.RWMutex
plugins map[string]interface{} // vendor → plugin instance
factories map[string]PluginFactory // "type_vendor" → factory
configs map[string]*PluginConfig // vendor → config
health map[string]*PluginHealth // vendor → health status
logger *slog.Logger
}
// NewPluginRegistry creates a new plugin registry.
func NewPluginRegistry() *PluginRegistry {
return &PluginRegistry{
plugins: make(map[string]interface{}),
factories: make(map[string]PluginFactory),
configs: make(map[string]*PluginConfig),
health: make(map[string]*PluginHealth),
logger: slog.Default().With("component", "shadow-ai-registry"),
}
}
// RegisterFactory registers a plugin factory for a given type+vendor combination.
// Example: RegisterFactory("firewall", "checkpoint", func() interface{} { return &CheckPointEnforcer{} })
func (r *PluginRegistry) RegisterFactory(pluginType PluginType, vendor string, factory PluginFactory) {
r.mu.Lock()
defer r.mu.Unlock()
key := fmt.Sprintf("%s_%s", pluginType, vendor)
r.factories[key] = factory
r.logger.Info("factory registered", "type", pluginType, "vendor", vendor)
}
// LoadPlugins creates and initializes plugins from configuration.
// Plugins that fail to initialize are logged but do not block other plugins.
func (r *PluginRegistry) LoadPlugins(config *IntegrationConfig) error {
r.mu.Lock()
defer r.mu.Unlock()
loaded := 0
for i := range config.Plugins {
pluginCfg := &config.Plugins[i]
if !pluginCfg.Enabled {
r.logger.Debug("plugin disabled, skipping", "vendor", pluginCfg.Vendor)
continue
}
key := fmt.Sprintf("%s_%s", pluginCfg.Type, pluginCfg.Vendor)
factory, exists := r.factories[key]
if !exists {
r.logger.Warn("no factory for plugin", "key", key, "vendor", pluginCfg.Vendor)
continue
}
plugin := factory()
// Initialize if plugin supports it.
if init, ok := plugin.(Initializer); ok {
if err := init.Initialize(pluginCfg.Config); err != nil {
r.logger.Error("plugin init failed", "vendor", pluginCfg.Vendor, "error", err)
continue
}
}
r.plugins[pluginCfg.Vendor] = plugin
r.configs[pluginCfg.Vendor] = pluginCfg
r.health[pluginCfg.Vendor] = &PluginHealth{
Vendor: pluginCfg.Vendor,
Type: pluginCfg.Type,
Status: PluginStatusHealthy,
}
loaded++
r.logger.Info("plugin loaded", "vendor", pluginCfg.Vendor, "type", pluginCfg.Type)
}
r.logger.Info("plugin loading complete", "loaded", loaded, "total", len(config.Plugins))
return nil
}
// Get returns a plugin by vendor name.
func (r *PluginRegistry) Get(vendor string) (interface{}, bool) {
r.mu.RLock()
defer r.mu.RUnlock()
p, ok := r.plugins[vendor]
return p, ok
}
// GetByType returns all plugins of a given type.
func (r *PluginRegistry) GetByType(pluginType PluginType) []interface{} {
r.mu.RLock()
defer r.mu.RUnlock()
var result []interface{}
for vendor, cfg := range r.configs {
if cfg.Type == pluginType {
if plugin, ok := r.plugins[vendor]; ok {
result = append(result, plugin)
}
}
}
return result
}
// GetNetworkEnforcers returns all loaded NetworkEnforcer plugins.
func (r *PluginRegistry) GetNetworkEnforcers() []NetworkEnforcer {
r.mu.RLock()
defer r.mu.RUnlock()
var result []NetworkEnforcer
for _, plugin := range r.plugins {
if ne, ok := plugin.(NetworkEnforcer); ok {
result = append(result, ne)
}
}
return result
}
// GetEndpointControllers returns all loaded EndpointController plugins.
func (r *PluginRegistry) GetEndpointControllers() []EndpointController {
r.mu.RLock()
defer r.mu.RUnlock()
var result []EndpointController
for _, plugin := range r.plugins {
if ec, ok := plugin.(EndpointController); ok {
result = append(result, ec)
}
}
return result
}
// GetWebGateways returns all loaded WebGateway plugins.
func (r *PluginRegistry) GetWebGateways() []WebGateway {
r.mu.RLock()
defer r.mu.RUnlock()
var result []WebGateway
for _, plugin := range r.plugins {
if wg, ok := plugin.(WebGateway); ok {
result = append(result, wg)
}
}
return result
}
// IsHealthy returns true if a plugin is currently healthy.
func (r *PluginRegistry) IsHealthy(vendor string) bool {
r.mu.RLock()
defer r.mu.RUnlock()
h, ok := r.health[vendor]
return ok && h.Status == PluginStatusHealthy
}
// SetHealth updates the health status for a plugin.
func (r *PluginRegistry) SetHealth(vendor string, health *PluginHealth) {
r.mu.Lock()
defer r.mu.Unlock()
r.health[vendor] = health
}
// GetHealth returns the health status snapshot for a plugin.
func (r *PluginRegistry) GetHealth(vendor string) (*PluginHealth, bool) {
r.mu.RLock()
defer r.mu.RUnlock()
h, ok := r.health[vendor]
if !ok {
return nil, false
}
cp := *h
return &cp, true
}
// AllHealth returns health snapshots for all plugins.
func (r *PluginRegistry) AllHealth() []PluginHealth {
r.mu.RLock()
defer r.mu.RUnlock()
result := make([]PluginHealth, 0, len(r.health))
for _, h := range r.health {
result = append(result, *h)
}
return result
}
// PluginCount returns the number of loaded plugins.
func (r *PluginRegistry) PluginCount() int {
r.mu.RLock()
defer r.mu.RUnlock()
return len(r.plugins)
}
// Vendors returns all loaded vendor names.
func (r *PluginRegistry) Vendors() []string {
r.mu.RLock()
defer r.mu.RUnlock()
result := make([]string, 0, len(r.plugins))
for v := range r.plugins {
result = append(result, v)
}
return result
}

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,373 @@
package shadow_ai
import (
"context"
"fmt"
"log/slog"
"sync"
"time"
)
// ShadowAIController is the main orchestrator that ties together
// detection, enforcement, SOC event emission, and statistics.
type ShadowAIController struct {
mu sync.RWMutex
registry *PluginRegistry
fallback *FallbackManager
healthChecker *HealthChecker
netDetector *NetworkDetector
behavioral *BehavioralDetector
docBridge *DocBridge
approval *ApprovalEngine
events []ShadowAIEvent // In-memory event store (bounded)
maxEvents int
socEventFn func(source, severity, category, description string, meta map[string]string) // Bridge to SOC event bus
logger *slog.Logger
}
// NewShadowAIController creates the main Shadow AI Control orchestrator.
func NewShadowAIController() *ShadowAIController {
registry := NewPluginRegistry()
RegisterDefaultPlugins(registry)
return &ShadowAIController{
registry: registry,
fallback: NewFallbackManager(registry, "detect_only"),
netDetector: NewNetworkDetector(),
behavioral: NewBehavioralDetector(100),
docBridge: NewDocBridge(),
approval: NewApprovalEngine(),
events: make([]ShadowAIEvent, 0, 1000),
maxEvents: 10000,
logger: slog.Default().With("component", "shadow-ai-controller"),
}
}
// SetSOCEventEmitter sets the function used to emit events into the SOC pipeline.
func (c *ShadowAIController) SetSOCEventEmitter(fn func(source, severity, category, description string, meta map[string]string)) {
c.mu.Lock()
defer c.mu.Unlock()
c.socEventFn = fn
}
// Configure loads plugin configuration and initializes the integration layer.
func (c *ShadowAIController) Configure(config *IntegrationConfig) error {
c.mu.Lock()
defer c.mu.Unlock()
if err := c.registry.LoadPlugins(config); err != nil {
return fmt.Errorf("failed to load plugins: %w", err)
}
c.fallback = NewFallbackManager(c.registry, config.FallbackStrategy)
c.fallback.SetEventLogger(func(event ShadowAIEvent) {
c.recordEvent(event)
})
interval := config.HealthCheckInterval
if interval <= 0 {
interval = 30 * time.Second
}
c.healthChecker = NewHealthChecker(c.registry, interval, func(vendor string, status PluginStatus, msg string) {
c.emitSOCEvent("HIGH", "integration_health", msg, map[string]string{
"vendor": vendor,
"status": string(status),
})
})
return nil
}
// StartHealthChecker starts continuous plugin health monitoring.
func (c *ShadowAIController) StartHealthChecker(ctx context.Context) {
if c.healthChecker != nil {
go c.healthChecker.Start(ctx)
}
}
// ProcessNetworkEvent analyzes a network event and enforces policy.
func (c *ShadowAIController) ProcessNetworkEvent(ctx context.Context, event NetworkEvent) *ShadowAIEvent {
detected := c.netDetector.Analyze(event)
if detected == nil {
return nil
}
// Record behavioral data.
c.behavioral.RecordAccess(event.User, event.Destination, event.DataSize)
// Attempt to block.
enforcedBy, err := c.fallback.BlockDomain(ctx, event.Destination, fmt.Sprintf("Shadow AI: %s", detected.AIService))
if err != nil {
c.logger.Error("enforcement failed", "destination", event.Destination, "error", err)
}
if enforcedBy != "" {
detected.Action = "blocked"
detected.EnforcedBy = enforcedBy
} else {
detected.Action = "detected"
}
detected.ID = genEventID()
c.recordEvent(*detected)
// Emit to SOC event bus.
c.emitSOCEvent("HIGH", "shadow_ai_usage",
fmt.Sprintf("Shadow AI access detected: %s → %s", event.User, detected.AIService),
map[string]string{
"user": event.User,
"hostname": event.Hostname,
"destination": event.Destination,
"ai_service": detected.AIService,
"action": detected.Action,
"enforced_by": detected.EnforcedBy,
},
)
return detected
}
// ScanContent scans text content for AI API keys.
func (c *ShadowAIController) ScanContent(content string) string {
return c.netDetector.SignatureDB().ScanForAPIKeys(content)
}
// ManualBlock manually blocks a domain or IP.
func (c *ShadowAIController) ManualBlock(ctx context.Context, req BlockRequest) error {
switch req.TargetType {
case "domain":
_, err := c.fallback.BlockDomain(ctx, req.Target, req.Reason)
return err
case "ip":
_, err := c.fallback.BlockIP(ctx, req.Target, req.Duration, req.Reason)
return err
case "host":
_, err := c.fallback.IsolateHost(ctx, req.Target)
return err
default:
return fmt.Errorf("unsupported target type: %s", req.TargetType)
}
}
// GetStats returns aggregate shadow AI statistics.
func (c *ShadowAIController) GetStats(timeRange string) ShadowAIStats {
c.mu.RLock()
defer c.mu.RUnlock()
cutoff := parseCutoff(timeRange)
stats := ShadowAIStats{
TimeRange: timeRange,
ByService: make(map[string]int),
ByDepartment: make(map[string]int),
}
violatorMap := make(map[string]int)
for _, e := range c.events {
if e.Timestamp.Before(cutoff) {
continue
}
stats.Total++
switch e.Action {
case "blocked":
stats.Blocked++
case "allowed", "approved":
stats.Approved++
case "pending":
stats.Pending++
}
if e.AIService != "" {
stats.ByService[e.AIService]++
}
if dept, ok := e.Metadata["department"]; ok {
stats.ByDepartment[dept]++
}
if e.UserID != "" {
violatorMap[e.UserID]++
}
}
// Build top violators list (sorted desc).
for uid, count := range violatorMap {
stats.TopViolators = append(stats.TopViolators, Violator{UserID: uid, Attempts: count})
}
// Sort by attempts descending, limit to 10.
for i := 0; i < len(stats.TopViolators); i++ {
for j := i + 1; j < len(stats.TopViolators); j++ {
if stats.TopViolators[j].Attempts > stats.TopViolators[i].Attempts {
stats.TopViolators[i], stats.TopViolators[j] = stats.TopViolators[j], stats.TopViolators[i]
}
}
}
if len(stats.TopViolators) > 10 {
stats.TopViolators = stats.TopViolators[:10]
}
return stats
}
// GetEvents returns recent shadow AI events (newest first).
func (c *ShadowAIController) GetEvents(limit int) []ShadowAIEvent {
c.mu.RLock()
defer c.mu.RUnlock()
total := len(c.events)
if total == 0 {
return nil
}
start := total - limit
if start < 0 {
start = 0
}
// Return newest first.
result := make([]ShadowAIEvent, 0, limit)
for i := total - 1; i >= start; i-- {
result = append(result, c.events[i])
}
return result
}
// GetEvent returns a single event by ID.
func (c *ShadowAIController) GetEvent(id string) (*ShadowAIEvent, bool) {
c.mu.RLock()
defer c.mu.RUnlock()
for i := len(c.events) - 1; i >= 0; i-- {
if c.events[i].ID == id {
cp := c.events[i]
return &cp, true
}
}
return nil, false
}
// IntegrationHealth returns health status of all plugins.
func (c *ShadowAIController) IntegrationHealth() []PluginHealth {
return c.registry.AllHealth()
}
// VendorHealth returns health for a specific vendor.
func (c *ShadowAIController) VendorHealth(vendor string) (*PluginHealth, bool) {
return c.registry.GetHealth(vendor)
}
// Registry returns the plugin registry for direct access.
func (c *ShadowAIController) Registry() *PluginRegistry {
return c.registry
}
// NetworkDetector returns the network detector for configuration.
func (c *ShadowAIController) NetworkDetector() *NetworkDetector {
return c.netDetector
}
// BehavioralDetector returns the behavioral detector.
func (c *ShadowAIController) BehavioralDetector() *BehavioralDetector {
return c.behavioral
}
// DocBridge returns the document review bridge.
func (c *ShadowAIController) DocBridge() *DocBridge {
return c.docBridge
}
// ApprovalEngine returns the approval workflow engine.
func (c *ShadowAIController) ApprovalEngine() *ApprovalEngine {
return c.approval
}
// ReviewDocument scans a document and creates an approval request if needed.
func (c *ShadowAIController) ReviewDocument(docID, content, userID string) (*ScanResult, *ApprovalRequest) {
result := c.docBridge.ScanDocument(docID, content, userID)
// Create approval request based on data classification.
var req *ApprovalRequest
if result.Status != DocReviewBlocked {
req = c.approval.SubmitRequest(userID, docID, result.DataClass)
}
// Emit SOC event for tracking.
c.emitSOCEvent("MEDIUM", "shadow_ai_usage",
fmt.Sprintf("Document review: %s by %s — %s (%s)",
docID, userID, result.Status, result.DataClass),
map[string]string{
"user": userID,
"doc_id": docID,
"status": string(result.Status),
"data_class": string(result.DataClass),
"pii_count": fmt.Sprintf("%d", len(result.PIIFound)),
},
)
return result, req
}
// GenerateComplianceReport generates a compliance report for the given period.
func (c *ShadowAIController) GenerateComplianceReport(period string) ComplianceReport {
stats := c.GetStats(period)
docStats := c.docBridge.Stats()
return ComplianceReport{
GeneratedAt: time.Now(),
Period: period,
TotalInteractions: stats.Total,
BlockedAttempts: stats.Blocked,
ApprovedReviews: stats.Approved,
PIIDetected: docStats["redacted"] + docStats["blocked"],
SecretsDetected: docStats["blocked"],
AuditComplete: true,
Regulations: []string{"GDPR", "SOC2", "EU AI Act Article 15"},
}
}
// --- Internal helpers ---
func (c *ShadowAIController) recordEvent(event ShadowAIEvent) {
c.mu.Lock()
defer c.mu.Unlock()
c.events = append(c.events, event)
// Evict oldest events if over capacity.
if len(c.events) > c.maxEvents {
excess := len(c.events) - c.maxEvents
c.events = c.events[excess:]
}
}
func (c *ShadowAIController) emitSOCEvent(severity, category, description string, meta map[string]string) {
c.mu.RLock()
fn := c.socEventFn
c.mu.RUnlock()
if fn != nil {
fn("shadow-ai", severity, category, description, meta)
}
}
func parseCutoff(timeRange string) time.Time {
switch timeRange {
case "1h":
return time.Now().Add(-1 * time.Hour)
case "24h":
return time.Now().Add(-24 * time.Hour)
case "7d":
return time.Now().Add(-7 * 24 * time.Hour)
case "30d":
return time.Now().Add(-30 * 24 * time.Hour)
case "90d":
return time.Now().Add(-90 * 24 * time.Hour)
default:
return time.Now().Add(-24 * time.Hour)
}
}
var eventCounter uint64
var eventCounterMu sync.Mutex
func genEventID() string {
eventCounterMu.Lock()
eventCounter++
id := eventCounter
eventCounterMu.Unlock()
return fmt.Sprintf("sai-%d-%d", time.Now().UnixMilli(), id)
}