mirror of
https://github.com/syntrex-lab/gomcp.git
synced 2026-04-27 21:36:21 +02:00
Release prep: 54 engines, self-hosted signatures, i18n, dashboard updates
This commit is contained in:
parent
694e32be26
commit
41cbfd6e0a
178 changed files with 36008 additions and 399 deletions
278
internal/application/shadow_ai/approval.go
Normal file
278
internal/application/shadow_ai/approval.go
Normal file
|
|
@ -0,0 +1,278 @@
|
|||
package shadow_ai
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// --- Tiered Approval Workflow ---
|
||||
// Implements §6 of the ТЗ: data classification → approval tier → SLA tracking.
|
||||
|
||||
// ApprovalStatus tracks the state of an approval request.
|
||||
type ApprovalStatus string
|
||||
|
||||
const (
|
||||
ApprovalPending ApprovalStatus = "pending"
|
||||
ApprovalApproved ApprovalStatus = "approved"
|
||||
ApprovalDenied ApprovalStatus = "denied"
|
||||
ApprovalExpired ApprovalStatus = "expired"
|
||||
ApprovalAutoApproved ApprovalStatus = "auto_approved"
|
||||
)
|
||||
|
||||
// DefaultApprovalTiers defines the approval requirements per data classification.
|
||||
func DefaultApprovalTiers() []ApprovalTier {
|
||||
return []ApprovalTier{
|
||||
{
|
||||
Name: "Tier 1: Public Data",
|
||||
DataClass: DataPublic,
|
||||
ApprovalNeeded: nil, // Auto-approve
|
||||
SLA: 0,
|
||||
AutoApprove: true,
|
||||
},
|
||||
{
|
||||
Name: "Tier 2: Internal Data",
|
||||
DataClass: DataInternal,
|
||||
ApprovalNeeded: []string{"manager"},
|
||||
SLA: 4 * time.Hour,
|
||||
AutoApprove: false,
|
||||
},
|
||||
{
|
||||
Name: "Tier 3: Confidential Data",
|
||||
DataClass: DataConfidential,
|
||||
ApprovalNeeded: []string{"manager", "soc"},
|
||||
SLA: 24 * time.Hour,
|
||||
AutoApprove: false,
|
||||
},
|
||||
{
|
||||
Name: "Tier 4: Critical Data",
|
||||
DataClass: DataCritical,
|
||||
ApprovalNeeded: []string{"ciso"},
|
||||
SLA: 0, // Manual only, no auto-expire
|
||||
AutoApprove: false,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// ApprovalEngine manages the tiered approval workflow.
|
||||
type ApprovalEngine struct {
|
||||
mu sync.RWMutex
|
||||
tiers []ApprovalTier
|
||||
requests map[string]*ApprovalRequest
|
||||
logger *slog.Logger
|
||||
}
|
||||
|
||||
// NewApprovalEngine creates an engine with default tiers.
|
||||
func NewApprovalEngine() *ApprovalEngine {
|
||||
return &ApprovalEngine{
|
||||
tiers: DefaultApprovalTiers(),
|
||||
requests: make(map[string]*ApprovalRequest),
|
||||
logger: slog.Default().With("component", "shadow-ai-approvals"),
|
||||
}
|
||||
}
|
||||
|
||||
// SubmitRequest creates a new approval request based on data classification.
|
||||
// Returns the request or auto-approves if the tier allows it.
|
||||
func (ae *ApprovalEngine) SubmitRequest(userID, docID string, dataClass DataClassification) *ApprovalRequest {
|
||||
ae.mu.Lock()
|
||||
defer ae.mu.Unlock()
|
||||
|
||||
tier := ae.findTier(dataClass)
|
||||
|
||||
req := &ApprovalRequest{
|
||||
ID: genApprovalID(),
|
||||
DocID: docID,
|
||||
UserID: userID,
|
||||
Tier: tier.Name,
|
||||
DataClass: dataClass,
|
||||
Status: string(ApprovalPending),
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
|
||||
// Set expiry based on SLA.
|
||||
if tier.SLA > 0 {
|
||||
req.ExpiresAt = req.CreatedAt.Add(tier.SLA)
|
||||
}
|
||||
|
||||
// Auto-approve for public data.
|
||||
if tier.AutoApprove {
|
||||
req.Status = string(ApprovalAutoApproved)
|
||||
req.ApprovedBy = "system"
|
||||
req.ResolvedAt = time.Now()
|
||||
ae.logger.Info("auto-approved",
|
||||
"request_id", req.ID,
|
||||
"user", userID,
|
||||
"data_class", dataClass,
|
||||
)
|
||||
} else {
|
||||
ae.logger.Info("approval required",
|
||||
"request_id", req.ID,
|
||||
"user", userID,
|
||||
"data_class", dataClass,
|
||||
"tier", tier.Name,
|
||||
"approvers", tier.ApprovalNeeded,
|
||||
)
|
||||
}
|
||||
|
||||
ae.requests[req.ID] = req
|
||||
return req
|
||||
}
|
||||
|
||||
// Approve approves a pending request.
|
||||
func (ae *ApprovalEngine) Approve(requestID, approvedBy string) error {
|
||||
ae.mu.Lock()
|
||||
defer ae.mu.Unlock()
|
||||
|
||||
req, ok := ae.requests[requestID]
|
||||
if !ok {
|
||||
return fmt.Errorf("request %s not found", requestID)
|
||||
}
|
||||
|
||||
if req.Status != string(ApprovalPending) {
|
||||
return fmt.Errorf("request %s is not pending (status: %s)", requestID, req.Status)
|
||||
}
|
||||
|
||||
req.Status = string(ApprovalApproved)
|
||||
req.ApprovedBy = approvedBy
|
||||
req.ResolvedAt = time.Now()
|
||||
|
||||
ae.logger.Info("approved",
|
||||
"request_id", requestID,
|
||||
"approved_by", approvedBy,
|
||||
)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Deny denies a pending request.
|
||||
func (ae *ApprovalEngine) Deny(requestID, deniedBy, reason string) error {
|
||||
ae.mu.Lock()
|
||||
defer ae.mu.Unlock()
|
||||
|
||||
req, ok := ae.requests[requestID]
|
||||
if !ok {
|
||||
return fmt.Errorf("request %s not found", requestID)
|
||||
}
|
||||
|
||||
if req.Status != string(ApprovalPending) {
|
||||
return fmt.Errorf("request %s is not pending (status: %s)", requestID, req.Status)
|
||||
}
|
||||
|
||||
req.Status = string(ApprovalDenied)
|
||||
req.DeniedBy = deniedBy
|
||||
req.Reason = reason
|
||||
req.ResolvedAt = time.Now()
|
||||
|
||||
ae.logger.Info("denied",
|
||||
"request_id", requestID,
|
||||
"denied_by", deniedBy,
|
||||
"reason", reason,
|
||||
)
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetRequest returns an approval request by ID.
|
||||
func (ae *ApprovalEngine) GetRequest(requestID string) (*ApprovalRequest, bool) {
|
||||
ae.mu.RLock()
|
||||
defer ae.mu.RUnlock()
|
||||
req, ok := ae.requests[requestID]
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
cp := *req
|
||||
return &cp, true
|
||||
}
|
||||
|
||||
// PendingRequests returns all pending approval requests.
|
||||
func (ae *ApprovalEngine) PendingRequests() []ApprovalRequest {
|
||||
ae.mu.RLock()
|
||||
defer ae.mu.RUnlock()
|
||||
|
||||
var result []ApprovalRequest
|
||||
for _, req := range ae.requests {
|
||||
if req.Status == string(ApprovalPending) {
|
||||
result = append(result, *req)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// ExpireOverdue marks overdue pending requests as expired.
|
||||
// Returns the number of expired requests.
|
||||
func (ae *ApprovalEngine) ExpireOverdue() int {
|
||||
ae.mu.Lock()
|
||||
defer ae.mu.Unlock()
|
||||
|
||||
now := time.Now()
|
||||
expired := 0
|
||||
|
||||
for _, req := range ae.requests {
|
||||
if req.Status == string(ApprovalPending) && !req.ExpiresAt.IsZero() && now.After(req.ExpiresAt) {
|
||||
req.Status = string(ApprovalExpired)
|
||||
req.ResolvedAt = now
|
||||
expired++
|
||||
ae.logger.Warn("request expired",
|
||||
"request_id", req.ID,
|
||||
"user", req.UserID,
|
||||
"expired_at", req.ExpiresAt,
|
||||
)
|
||||
}
|
||||
}
|
||||
return expired
|
||||
}
|
||||
|
||||
// Stats returns approval workflow statistics.
|
||||
func (ae *ApprovalEngine) Stats() map[string]int {
|
||||
ae.mu.RLock()
|
||||
defer ae.mu.RUnlock()
|
||||
|
||||
stats := map[string]int{
|
||||
"total": len(ae.requests),
|
||||
"pending": 0,
|
||||
"approved": 0,
|
||||
"denied": 0,
|
||||
"expired": 0,
|
||||
"auto_approved": 0,
|
||||
}
|
||||
for _, req := range ae.requests {
|
||||
switch ApprovalStatus(req.Status) {
|
||||
case ApprovalPending:
|
||||
stats["pending"]++
|
||||
case ApprovalApproved:
|
||||
stats["approved"]++
|
||||
case ApprovalDenied:
|
||||
stats["denied"]++
|
||||
case ApprovalExpired:
|
||||
stats["expired"]++
|
||||
case ApprovalAutoApproved:
|
||||
stats["auto_approved"]++
|
||||
}
|
||||
}
|
||||
return stats
|
||||
}
|
||||
|
||||
// Tiers returns the approval tier configuration.
|
||||
func (ae *ApprovalEngine) Tiers() []ApprovalTier {
|
||||
return ae.tiers
|
||||
}
|
||||
|
||||
func (ae *ApprovalEngine) findTier(dataClass DataClassification) ApprovalTier {
|
||||
for _, t := range ae.tiers {
|
||||
if t.DataClass == dataClass {
|
||||
return t
|
||||
}
|
||||
}
|
||||
// Default to most restrictive.
|
||||
return ae.tiers[len(ae.tiers)-1]
|
||||
}
|
||||
|
||||
var approvalCounter uint64
|
||||
var approvalCounterMu sync.Mutex
|
||||
|
||||
func genApprovalID() string {
|
||||
approvalCounterMu.Lock()
|
||||
approvalCounter++
|
||||
id := approvalCounter
|
||||
approvalCounterMu.Unlock()
|
||||
return fmt.Sprintf("apr-%d-%d", time.Now().UnixMilli(), id)
|
||||
}
|
||||
116
internal/application/shadow_ai/correlation.go
Normal file
116
internal/application/shadow_ai/correlation.go
Normal file
|
|
@ -0,0 +1,116 @@
|
|||
package shadow_ai
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
domsoc "github.com/syntrex/gomcp/internal/domain/soc"
|
||||
)
|
||||
|
||||
// ShadowAICorrelationRules returns SOC correlation rules specific to Shadow AI
|
||||
// detection. These integrate into the existing SOC correlation engine.
|
||||
func ShadowAICorrelationRules() []domsoc.SOCCorrelationRule {
|
||||
return []domsoc.SOCCorrelationRule{
|
||||
{
|
||||
ID: "SAI-CR-001",
|
||||
Name: "Multi-Service Shadow AI",
|
||||
RequiredCategories: []string{"shadow_ai_usage"},
|
||||
MinEvents: 3,
|
||||
TimeWindow: 10 * time.Minute,
|
||||
Severity: domsoc.SeverityHigh,
|
||||
KillChainPhase: "Reconnaissance",
|
||||
MITREMapping: []string{"T1595"},
|
||||
Description: "User accessing 3+ distinct AI services within 10 minutes. Indicates active AI tool exploration or data shopping across providers.",
|
||||
},
|
||||
{
|
||||
ID: "SAI-CR-002",
|
||||
Name: "Shadow AI + Data Exfiltration",
|
||||
RequiredCategories: []string{"shadow_ai_usage", "exfiltration"},
|
||||
MinEvents: 2,
|
||||
TimeWindow: 15 * time.Minute,
|
||||
Severity: domsoc.SeverityCritical,
|
||||
KillChainPhase: "Exfiltration",
|
||||
MITREMapping: []string{"T1041", "T1567"},
|
||||
Description: "Shadow AI usage followed by data exfiltration attempt. Possible corporate data leakage via unauthorized AI services.",
|
||||
},
|
||||
{
|
||||
ID: "SAI-CR-003",
|
||||
Name: "Shadow AI Volume Spike",
|
||||
RequiredCategories: []string{"shadow_ai_usage"},
|
||||
MinEvents: 10,
|
||||
TimeWindow: 1 * time.Hour,
|
||||
Severity: domsoc.SeverityHigh,
|
||||
KillChainPhase: "Actions on Objectives",
|
||||
MITREMapping: []string{"T1048"},
|
||||
Description: "10+ shadow AI events from same source within 1 hour. Indicates bulk data transfer to external AI service.",
|
||||
},
|
||||
{
|
||||
ID: "SAI-CR-004",
|
||||
Name: "Shadow AI After Hours",
|
||||
RequiredCategories: []string{"shadow_ai_usage"},
|
||||
MinEvents: 2,
|
||||
TimeWindow: 30 * time.Minute,
|
||||
Severity: domsoc.SeverityMedium,
|
||||
KillChainPhase: "Persistence",
|
||||
MITREMapping: []string{"T1053"},
|
||||
Description: "Shadow AI usage outside business hours (detected via timestamp clustering). May indicate automated scripts or insider threat.",
|
||||
},
|
||||
{
|
||||
ID: "SAI-CR-005",
|
||||
Name: "Integration Failure Chain",
|
||||
RequiredCategories: []string{"integration_health"},
|
||||
MinEvents: 3,
|
||||
TimeWindow: 5 * time.Minute,
|
||||
Severity: domsoc.SeverityCritical,
|
||||
KillChainPhase: "Defense Evasion",
|
||||
MITREMapping: []string{"T1562"},
|
||||
Description: "3+ integration health failures in 5 minutes. Possible attack on enforcement infrastructure to blind Shadow AI detection.",
|
||||
},
|
||||
{
|
||||
ID: "SAI-CR-006",
|
||||
Name: "Shadow AI + PII Leak",
|
||||
RequiredCategories: []string{"shadow_ai_usage", "pii_leak"},
|
||||
MinEvents: 2,
|
||||
TimeWindow: 10 * time.Minute,
|
||||
Severity: domsoc.SeverityCritical,
|
||||
KillChainPhase: "Exfiltration",
|
||||
MITREMapping: []string{"T1567.002"},
|
||||
Description: "Shadow AI usage combined with PII leak detection. GDPR/regulatory violation in progress — immediate response required.",
|
||||
},
|
||||
{
|
||||
ID: "SAI-CR-007",
|
||||
Name: "Shadow AI Evasion Attempt",
|
||||
SequenceCategories: []string{"shadow_ai_usage", "evasion"},
|
||||
MinEvents: 2,
|
||||
TimeWindow: 10 * time.Minute,
|
||||
Severity: domsoc.SeverityHigh,
|
||||
KillChainPhase: "Defense Evasion",
|
||||
MITREMapping: []string{"T1090", "T1573"},
|
||||
Description: "Shadow AI usage followed by evasion technique (VPN, proxy chaining, encoding). User attempting to bypass detection.",
|
||||
},
|
||||
{
|
||||
ID: "SAI-CR-008",
|
||||
Name: "Cross-Department AI Usage",
|
||||
RequiredCategories: []string{"shadow_ai_usage"},
|
||||
MinEvents: 5,
|
||||
TimeWindow: 30 * time.Minute,
|
||||
Severity: domsoc.SeverityMedium,
|
||||
CrossSensor: true,
|
||||
KillChainPhase: "Lateral Movement",
|
||||
MITREMapping: []string{"T1021"},
|
||||
Description: "Shadow AI events from 5+ distinct network segments/sensors within 30 minutes. Indicates coordinated policy circumvention or compromised credentials used across departments.",
|
||||
},
|
||||
// Severity trend: escalating shadow AI event severity
|
||||
{
|
||||
ID: "SAI-CR-009",
|
||||
Name: "Shadow AI Escalation",
|
||||
SeverityTrend: "ascending",
|
||||
TrendCategory: "shadow_ai_usage",
|
||||
MinEvents: 3,
|
||||
TimeWindow: 30 * time.Minute,
|
||||
Severity: domsoc.SeverityCritical,
|
||||
KillChainPhase: "Exploitation",
|
||||
MITREMapping: []string{"T1059"},
|
||||
Description: "Ascending severity pattern in Shadow AI events: user escalating from casual browsing to bulk data uploads. Crescendo data theft in progress.",
|
||||
},
|
||||
}
|
||||
}
|
||||
503
internal/application/shadow_ai/detection.go
Normal file
503
internal/application/shadow_ai/detection.go
Normal file
|
|
@ -0,0 +1,503 @@
|
|||
package shadow_ai
|
||||
|
||||
import (
|
||||
"log/slog"
|
||||
"math"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// --- AI Signature Database ---
|
||||
|
||||
// AISignatureDB contains known AI service signatures for detection.
|
||||
type AISignatureDB struct {
|
||||
mu sync.RWMutex
|
||||
services []AIServiceInfo
|
||||
domainPatterns []*domainPattern
|
||||
apiKeyPatterns []*APIKeyPattern
|
||||
httpSignatures []string
|
||||
}
|
||||
|
||||
type domainPattern struct {
|
||||
original string
|
||||
regex *regexp.Regexp
|
||||
service string
|
||||
}
|
||||
|
||||
// APIKeyPattern defines a regex pattern for detecting AI API keys.
|
||||
type APIKeyPattern struct {
|
||||
Name string `json:"name"`
|
||||
Pattern *regexp.Regexp `json:"-"`
|
||||
Entropy float64 `json:"min_entropy"`
|
||||
}
|
||||
|
||||
// NewAISignatureDB creates a signature database pre-loaded with known AI services.
|
||||
func NewAISignatureDB() *AISignatureDB {
|
||||
db := &AISignatureDB{}
|
||||
db.loadDefaults()
|
||||
return db
|
||||
}
|
||||
|
||||
// loadDefaults populates the database with known AI services and patterns.
|
||||
func (db *AISignatureDB) loadDefaults() {
|
||||
db.services = defaultAIServices()
|
||||
|
||||
// Compile domain patterns.
|
||||
for _, svc := range db.services {
|
||||
for _, d := range svc.Domains {
|
||||
pattern := domainToRegex(d)
|
||||
db.domainPatterns = append(db.domainPatterns, &domainPattern{
|
||||
original: d,
|
||||
regex: pattern,
|
||||
service: svc.Name,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// API key patterns.
|
||||
db.apiKeyPatterns = defaultAPIKeyPatterns()
|
||||
|
||||
// HTTP header signatures.
|
||||
db.httpSignatures = []string{
|
||||
"authorization: bearer sk-", // OpenAI
|
||||
"authorization: bearer ant-", // Anthropic
|
||||
"x-api-key: sk-ant-", // Anthropic v2
|
||||
"x-goog-api-key:", // Google AI
|
||||
"authorization: bearer gsk_", // Groq
|
||||
"authorization: bearer hf_", // HuggingFace
|
||||
}
|
||||
}
|
||||
|
||||
// domainToRegex converts a wildcard domain (e.g., "*.openai.com") to a regex.
|
||||
func domainToRegex(domain string) *regexp.Regexp {
|
||||
escaped := regexp.QuoteMeta(domain)
|
||||
escaped = strings.ReplaceAll(escaped, `\*`, `[a-zA-Z0-9\-]+`)
|
||||
return regexp.MustCompile("(?i)^" + escaped + "$")
|
||||
}
|
||||
|
||||
// MatchDomain checks if a domain matches any known AI service.
|
||||
// Returns the service name or empty string.
|
||||
func (db *AISignatureDB) MatchDomain(domain string) string {
|
||||
db.mu.RLock()
|
||||
defer db.mu.RUnlock()
|
||||
|
||||
domain = strings.ToLower(strings.TrimSpace(domain))
|
||||
for _, dp := range db.domainPatterns {
|
||||
if dp.regex.MatchString(domain) {
|
||||
return dp.service
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// MatchHTTPHeaders checks if HTTP headers contain known AI service signatures.
|
||||
func (db *AISignatureDB) MatchHTTPHeaders(headers map[string]string) string {
|
||||
db.mu.RLock()
|
||||
defer db.mu.RUnlock()
|
||||
|
||||
for key, value := range headers {
|
||||
headerLine := strings.ToLower(key + ": " + value)
|
||||
for _, sig := range db.httpSignatures {
|
||||
if strings.Contains(headerLine, sig) {
|
||||
return sig
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// ScanForAPIKeys scans content for AI API keys.
|
||||
// Returns the matched pattern name or empty string.
|
||||
func (db *AISignatureDB) ScanForAPIKeys(content string) string {
|
||||
db.mu.RLock()
|
||||
defer db.mu.RUnlock()
|
||||
|
||||
for _, pattern := range db.apiKeyPatterns {
|
||||
if pattern.Pattern.MatchString(content) {
|
||||
return pattern.Name
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// ServiceCount returns the number of known AI services.
|
||||
func (db *AISignatureDB) ServiceCount() int {
|
||||
db.mu.RLock()
|
||||
defer db.mu.RUnlock()
|
||||
return len(db.services)
|
||||
}
|
||||
|
||||
// DomainPatternCount returns the number of compiled domain patterns.
|
||||
func (db *AISignatureDB) DomainPatternCount() int {
|
||||
db.mu.RLock()
|
||||
defer db.mu.RUnlock()
|
||||
return len(db.domainPatterns)
|
||||
}
|
||||
|
||||
// AddService adds a custom AI service to the database.
|
||||
func (db *AISignatureDB) AddService(svc AIServiceInfo) {
|
||||
db.mu.Lock()
|
||||
defer db.mu.Unlock()
|
||||
|
||||
db.services = append(db.services, svc)
|
||||
for _, d := range svc.Domains {
|
||||
pattern := domainToRegex(d)
|
||||
db.domainPatterns = append(db.domainPatterns, &domainPattern{
|
||||
original: d,
|
||||
regex: pattern,
|
||||
service: svc.Name,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// --- Network Detector ---
|
||||
|
||||
// NetworkEvent represents a network connection event for analysis.
|
||||
type NetworkEvent struct {
|
||||
User string `json:"user"`
|
||||
Hostname string `json:"hostname"`
|
||||
Destination string `json:"destination"` // Domain or IP
|
||||
Port int `json:"port"`
|
||||
HTTPHeaders map[string]string `json:"http_headers,omitempty"`
|
||||
TLSJA3 string `json:"tls_ja3,omitempty"`
|
||||
DataSize int64 `json:"data_size"`
|
||||
Timestamp time.Time `json:"timestamp"`
|
||||
}
|
||||
|
||||
// NetworkDetector analyzes network events for AI service access.
|
||||
type NetworkDetector struct {
|
||||
signatures *AISignatureDB
|
||||
logger *slog.Logger
|
||||
}
|
||||
|
||||
// NewNetworkDetector creates a new network detector with the default signature DB.
|
||||
func NewNetworkDetector() *NetworkDetector {
|
||||
return &NetworkDetector{
|
||||
signatures: NewAISignatureDB(),
|
||||
logger: slog.Default().With("component", "shadow-ai-network"),
|
||||
}
|
||||
}
|
||||
|
||||
// NewNetworkDetectorWithDB creates a detector with a custom signature database.
|
||||
func NewNetworkDetectorWithDB(db *AISignatureDB) *NetworkDetector {
|
||||
return &NetworkDetector{
|
||||
signatures: db,
|
||||
logger: slog.Default().With("component", "shadow-ai-network"),
|
||||
}
|
||||
}
|
||||
|
||||
// Analyze checks a network event for AI service access.
|
||||
// Returns a ShadowAIEvent if detected, nil otherwise.
|
||||
func (nd *NetworkDetector) Analyze(event NetworkEvent) *ShadowAIEvent {
|
||||
// Check domain match.
|
||||
if service := nd.signatures.MatchDomain(event.Destination); service != "" {
|
||||
nd.logger.Info("AI domain detected",
|
||||
"user", event.User,
|
||||
"destination", event.Destination,
|
||||
"service", service,
|
||||
)
|
||||
return &ShadowAIEvent{
|
||||
UserID: event.User,
|
||||
Hostname: event.Hostname,
|
||||
Destination: event.Destination,
|
||||
AIService: service,
|
||||
DetectionMethod: DetectNetwork,
|
||||
Action: "detected",
|
||||
DataSize: event.DataSize,
|
||||
Timestamp: event.Timestamp,
|
||||
}
|
||||
}
|
||||
|
||||
// Check HTTP header signatures.
|
||||
if sig := nd.signatures.MatchHTTPHeaders(event.HTTPHeaders); sig != "" {
|
||||
nd.logger.Info("AI HTTP signature detected",
|
||||
"user", event.User,
|
||||
"destination", event.Destination,
|
||||
"signature", sig,
|
||||
)
|
||||
return &ShadowAIEvent{
|
||||
UserID: event.User,
|
||||
Hostname: event.Hostname,
|
||||
Destination: event.Destination,
|
||||
AIService: "unknown",
|
||||
DetectionMethod: DetectHTTP,
|
||||
Action: "detected",
|
||||
DataSize: event.DataSize,
|
||||
Timestamp: event.Timestamp,
|
||||
Metadata: map[string]string{"http_signature": sig},
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// SignatureDB returns the underlying signature database for extension.
|
||||
func (nd *NetworkDetector) SignatureDB() *AISignatureDB {
|
||||
return nd.signatures
|
||||
}
|
||||
|
||||
// --- Behavioral Detector ---
|
||||
|
||||
// UserBehaviorProfile tracks a user's AI access behavior for anomaly detection.
|
||||
type UserBehaviorProfile struct {
|
||||
UserID string `json:"user_id"`
|
||||
AccessFrequency float64 `json:"access_frequency"` // Requests per hour
|
||||
DataVolumePerHour float64 `json:"data_volume_per_hour"` // Bytes per hour
|
||||
KnownDestinations []string `json:"known_destinations"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
// BehavioralAlert is emitted when anomalous AI access is detected.
|
||||
type BehavioralAlert struct {
|
||||
UserID string `json:"user_id"`
|
||||
AnomalyType string `json:"anomaly_type"` // "access_spike", "new_destination", "data_volume_spike"
|
||||
Current float64 `json:"current"`
|
||||
Baseline float64 `json:"baseline"`
|
||||
ZScore float64 `json:"z_score"`
|
||||
Destination string `json:"destination,omitempty"`
|
||||
Severity string `json:"severity"`
|
||||
}
|
||||
|
||||
// BehavioralDetector detects anomalous AI usage patterns per user.
|
||||
type BehavioralDetector struct {
|
||||
mu sync.RWMutex
|
||||
baselines map[string]*UserBehaviorProfile
|
||||
current map[string]*UserBehaviorProfile
|
||||
alertBus chan BehavioralAlert
|
||||
logger *slog.Logger
|
||||
}
|
||||
|
||||
// NewBehavioralDetector creates a behavioral detector with a buffered alert bus.
|
||||
func NewBehavioralDetector(alertBufSize int) *BehavioralDetector {
|
||||
if alertBufSize <= 0 {
|
||||
alertBufSize = 100
|
||||
}
|
||||
return &BehavioralDetector{
|
||||
baselines: make(map[string]*UserBehaviorProfile),
|
||||
current: make(map[string]*UserBehaviorProfile),
|
||||
alertBus: make(chan BehavioralAlert, alertBufSize),
|
||||
logger: slog.Default().With("component", "shadow-ai-behavioral"),
|
||||
}
|
||||
}
|
||||
|
||||
// RecordAccess records a single AI access attempt for behavioral tracking.
|
||||
func (bd *BehavioralDetector) RecordAccess(userID, destination string, dataSize int64) {
|
||||
bd.mu.Lock()
|
||||
defer bd.mu.Unlock()
|
||||
|
||||
profile, ok := bd.current[userID]
|
||||
if !ok {
|
||||
profile = &UserBehaviorProfile{
|
||||
UserID: userID,
|
||||
}
|
||||
bd.current[userID] = profile
|
||||
}
|
||||
|
||||
profile.AccessFrequency++
|
||||
profile.DataVolumePerHour += float64(dataSize)
|
||||
profile.UpdatedAt = time.Now()
|
||||
|
||||
// Track destinations.
|
||||
found := false
|
||||
for _, d := range profile.KnownDestinations {
|
||||
if d == destination {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
profile.KnownDestinations = append(profile.KnownDestinations, destination)
|
||||
}
|
||||
}
|
||||
|
||||
// SetBaseline sets the known baseline behavior for a user.
|
||||
func (bd *BehavioralDetector) SetBaseline(userID string, profile *UserBehaviorProfile) {
|
||||
bd.mu.Lock()
|
||||
defer bd.mu.Unlock()
|
||||
bd.baselines[userID] = profile
|
||||
}
|
||||
|
||||
// DetectAnomalies compares current behavior to baselines and emits alerts.
|
||||
func (bd *BehavioralDetector) DetectAnomalies() []BehavioralAlert {
|
||||
bd.mu.RLock()
|
||||
defer bd.mu.RUnlock()
|
||||
|
||||
var alerts []BehavioralAlert
|
||||
|
||||
for userID, current := range bd.current {
|
||||
baseline, ok := bd.baselines[userID]
|
||||
if !ok {
|
||||
// No baseline — any AI access from this user is suspicious.
|
||||
if current.AccessFrequency > 0 {
|
||||
alert := BehavioralAlert{
|
||||
UserID: userID,
|
||||
AnomalyType: "first_ai_access",
|
||||
Current: current.AccessFrequency,
|
||||
Baseline: 0,
|
||||
Severity: "WARNING",
|
||||
}
|
||||
alerts = append(alerts, alert)
|
||||
bd.emitAlert(alert)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
// Z-score for access frequency.
|
||||
if baseline.AccessFrequency > 0 {
|
||||
zscore := (current.AccessFrequency - baseline.AccessFrequency) / math.Max(baseline.AccessFrequency*0.3, 1)
|
||||
if math.Abs(zscore) > 3.0 {
|
||||
alert := BehavioralAlert{
|
||||
UserID: userID,
|
||||
AnomalyType: "access_spike",
|
||||
Current: current.AccessFrequency,
|
||||
Baseline: baseline.AccessFrequency,
|
||||
ZScore: zscore,
|
||||
Severity: "WARNING",
|
||||
}
|
||||
alerts = append(alerts, alert)
|
||||
bd.emitAlert(alert)
|
||||
}
|
||||
}
|
||||
|
||||
// Detect new AI destinations.
|
||||
for _, dest := range current.KnownDestinations {
|
||||
isNew := true
|
||||
for _, known := range baseline.KnownDestinations {
|
||||
if dest == known {
|
||||
isNew = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if isNew {
|
||||
alert := BehavioralAlert{
|
||||
UserID: userID,
|
||||
AnomalyType: "new_ai_destination",
|
||||
Destination: dest,
|
||||
Severity: "HIGH",
|
||||
}
|
||||
alerts = append(alerts, alert)
|
||||
bd.emitAlert(alert)
|
||||
}
|
||||
}
|
||||
|
||||
// Z-score for data volume.
|
||||
if baseline.DataVolumePerHour > 0 {
|
||||
zscore := (current.DataVolumePerHour - baseline.DataVolumePerHour) / math.Max(baseline.DataVolumePerHour*0.3, 1)
|
||||
if math.Abs(zscore) > 3.0 {
|
||||
alert := BehavioralAlert{
|
||||
UserID: userID,
|
||||
AnomalyType: "data_volume_spike",
|
||||
Current: current.DataVolumePerHour,
|
||||
Baseline: baseline.DataVolumePerHour,
|
||||
ZScore: zscore,
|
||||
Severity: "CRITICAL",
|
||||
}
|
||||
alerts = append(alerts, alert)
|
||||
bd.emitAlert(alert)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return alerts
|
||||
}
|
||||
|
||||
// Alerts returns the alert channel for consuming behavioral alerts.
|
||||
func (bd *BehavioralDetector) Alerts() <-chan BehavioralAlert {
|
||||
return bd.alertBus
|
||||
}
|
||||
|
||||
// ResetCurrent clears the current period data (call after each analysis window).
|
||||
func (bd *BehavioralDetector) ResetCurrent() {
|
||||
bd.mu.Lock()
|
||||
defer bd.mu.Unlock()
|
||||
bd.current = make(map[string]*UserBehaviorProfile)
|
||||
}
|
||||
|
||||
func (bd *BehavioralDetector) emitAlert(alert BehavioralAlert) {
|
||||
select {
|
||||
case bd.alertBus <- alert:
|
||||
default:
|
||||
bd.logger.Warn("behavioral alert bus full, dropping alert",
|
||||
"user", alert.UserID,
|
||||
"type", alert.AnomalyType,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// --- Default Data ---
|
||||
|
||||
func defaultAIServices() []AIServiceInfo {
|
||||
return []AIServiceInfo{
|
||||
{Name: "ChatGPT", Vendor: "OpenAI", Domains: []string{"chat.openai.com", "api.openai.com", "*.openai.com"}, Category: "llm"},
|
||||
{Name: "Claude", Vendor: "Anthropic", Domains: []string{"claude.ai", "api.anthropic.com", "*.anthropic.com"}, Category: "llm"},
|
||||
{Name: "Gemini", Vendor: "Google", Domains: []string{"gemini.google.com", "generativelanguage.googleapis.com", "aistudio.google.com"}, Category: "llm"},
|
||||
{Name: "Copilot", Vendor: "Microsoft", Domains: []string{"copilot.microsoft.com", "*.copilot.microsoft.com"}, Category: "code_assist"},
|
||||
{Name: "Cohere", Vendor: "Cohere", Domains: []string{"api.cohere.ai", "dashboard.cohere.com", "*.cohere.ai"}, Category: "llm"},
|
||||
{Name: "AI21", Vendor: "AI21 Labs", Domains: []string{"api.ai21.com", "studio.ai21.com", "*.ai21.com"}, Category: "llm"},
|
||||
{Name: "HuggingFace", Vendor: "Hugging Face", Domains: []string{"api-inference.huggingface.co", "huggingface.co", "*.huggingface.co"}, Category: "llm"},
|
||||
{Name: "Replicate", Vendor: "Replicate", Domains: []string{"api.replicate.com", "replicate.com", "*.replicate.com"}, Category: "llm"},
|
||||
{Name: "Mistral", Vendor: "Mistral AI", Domains: []string{"api.mistral.ai", "chat.mistral.ai", "*.mistral.ai"}, Category: "llm"},
|
||||
{Name: "Perplexity", Vendor: "Perplexity", Domains: []string{"api.perplexity.ai", "perplexity.ai", "*.perplexity.ai"}, Category: "llm"},
|
||||
{Name: "Groq", Vendor: "Groq", Domains: []string{"api.groq.com", "groq.com", "*.groq.com"}, Category: "llm"},
|
||||
{Name: "Together", Vendor: "Together AI", Domains: []string{"api.together.xyz", "together.ai", "*.together.ai"}, Category: "llm"},
|
||||
{Name: "Stability", Vendor: "Stability AI", Domains: []string{"api.stability.ai", "*.stability.ai"}, Category: "image_gen"},
|
||||
{Name: "Midjourney", Vendor: "Midjourney", Domains: []string{"midjourney.com", "*.midjourney.com"}, Category: "image_gen"},
|
||||
{Name: "DALL-E", Vendor: "OpenAI", Domains: []string{"labs.openai.com"}, Category: "image_gen"},
|
||||
{Name: "Cursor", Vendor: "Cursor", Domains: []string{"api2.cursor.sh", "*.cursor.sh"}, Category: "code_assist"},
|
||||
{Name: "Replit AI", Vendor: "Replit", Domains: []string{"replit.com", "*.replit.com"}, Category: "code_assist"},
|
||||
{Name: "Codeium", Vendor: "Codeium", Domains: []string{"*.codeium.com", "codeium.com"}, Category: "code_assist"},
|
||||
{Name: "Tabnine", Vendor: "Tabnine", Domains: []string{"*.tabnine.com", "tabnine.com"}, Category: "code_assist"},
|
||||
{Name: "Qwen", Vendor: "Alibaba", Domains: []string{"dashscope.aliyuncs.com", "*.dashscope.aliyuncs.com"}, Category: "llm"},
|
||||
{Name: "DeepSeek", Vendor: "DeepSeek", Domains: []string{"api.deepseek.com", "chat.deepseek.com", "*.deepseek.com"}, Category: "llm"},
|
||||
{Name: "Kimi", Vendor: "Moonshot AI", Domains: []string{"api.moonshot.cn", "kimi.moonshot.cn", "*.moonshot.cn"}, Category: "llm"},
|
||||
{Name: "Baidu ERNIE", Vendor: "Baidu", Domains: []string{"aip.baidubce.com", "erniebot.baidu.com"}, Category: "llm"},
|
||||
{Name: "Jasper", Vendor: "Jasper", Domains: []string{"app.jasper.ai", "api.jasper.ai", "*.jasper.ai"}, Category: "llm"},
|
||||
{Name: "Writer", Vendor: "Writer", Domains: []string{"writer.com", "api.writer.com", "*.writer.com"}, Category: "llm"},
|
||||
{Name: "Notion AI", Vendor: "Notion", Domains: []string{"www.notion.so"}, Category: "productivity"},
|
||||
{Name: "Grammarly AI", Vendor: "Grammarly", Domains: []string{"*.grammarly.com"}, Category: "productivity"},
|
||||
{Name: "Runway", Vendor: "Runway", Domains: []string{"app.runwayml.com", "api.runwayml.com", "*.runwayml.com"}, Category: "video_gen"},
|
||||
{Name: "Pika", Vendor: "Pika", Domains: []string{"pika.art", "*.pika.art"}, Category: "video_gen"},
|
||||
{Name: "ElevenLabs", Vendor: "ElevenLabs", Domains: []string{"api.elevenlabs.io", "elevenlabs.io", "*.elevenlabs.io"}, Category: "audio_gen"},
|
||||
{Name: "Suno", Vendor: "Suno", Domains: []string{"suno.com", "*.suno.com"}, Category: "audio_gen"},
|
||||
{Name: "OpenRouter", Vendor: "OpenRouter", Domains: []string{"openrouter.ai", "*.openrouter.ai"}, Category: "llm"},
|
||||
{Name: "Scale AI", Vendor: "Scale", Domains: []string{"scale.com", "api.scale.com", "*.scale.com"}, Category: "llm"},
|
||||
{Name: "Inflection Pi", Vendor: "Inflection", Domains: []string{"pi.ai", "api.inflection.ai"}, Category: "llm"},
|
||||
{Name: "Grok", Vendor: "xAI", Domains: []string{"grok.x.ai", "api.x.ai"}, Category: "llm"},
|
||||
{Name: "Character.AI", Vendor: "Character.AI", Domains: []string{"character.ai", "*.character.ai"}, Category: "llm"},
|
||||
{Name: "Poe", Vendor: "Quora", Domains: []string{"poe.com", "*.poe.com"}, Category: "llm"},
|
||||
{Name: "You.com", Vendor: "You.com", Domains: []string{"you.com", "api.you.com"}, Category: "llm"},
|
||||
{Name: "Phind", Vendor: "Phind", Domains: []string{"phind.com", "*.phind.com"}, Category: "llm"},
|
||||
}
|
||||
}
|
||||
|
||||
func defaultAPIKeyPatterns() []*APIKeyPattern {
|
||||
return []*APIKeyPattern{
|
||||
{Name: "OpenAI API Key", Pattern: regexp.MustCompile(`sk-[a-zA-Z0-9]{20,}T3BlbkFJ[a-zA-Z0-9]{20,}`), Entropy: 4.5},
|
||||
{Name: "OpenAI Project Key", Pattern: regexp.MustCompile(`sk-proj-[a-zA-Z0-9\-_]{48,}`), Entropy: 4.5},
|
||||
{Name: "Anthropic API Key", Pattern: regexp.MustCompile(`sk-ant-[a-zA-Z0-9\-_]{90,}`), Entropy: 4.5},
|
||||
{Name: "Google AI API Key", Pattern: regexp.MustCompile(`AIza[0-9A-Za-z\-_]{35}`), Entropy: 4.0},
|
||||
{Name: "HuggingFace Token", Pattern: regexp.MustCompile(`hf_[a-zA-Z0-9]{34}`), Entropy: 4.5},
|
||||
{Name: "Groq API Key", Pattern: regexp.MustCompile(`gsk_[a-zA-Z0-9]{52}`), Entropy: 4.5},
|
||||
{Name: "Cohere API Key", Pattern: regexp.MustCompile(`[a-zA-Z0-9]{10,}-[a-zA-Z0-9]{4,}-[a-zA-Z0-9]{4,}-[a-zA-Z0-9]{4,}-[a-zA-Z0-9]{12,}`), Entropy: 4.5},
|
||||
{Name: "Replicate API Token", Pattern: regexp.MustCompile(`r8_[a-zA-Z0-9]{37}`), Entropy: 4.5},
|
||||
}
|
||||
}
|
||||
|
||||
// ServicesByCategory returns AI services grouped by category.
|
||||
func ServicesByCategory() map[string][]AIServiceInfo {
|
||||
services := defaultAIServices()
|
||||
result := make(map[string][]AIServiceInfo)
|
||||
for _, svc := range services {
|
||||
result[svc.Category] = append(result[svc.Category], svc)
|
||||
}
|
||||
// Sort each category by name for deterministic output.
|
||||
for cat := range result {
|
||||
sort.Slice(result[cat], func(i, j int) bool {
|
||||
return result[cat][i].Name < result[cat][j].Name
|
||||
})
|
||||
}
|
||||
return result
|
||||
}
|
||||
353
internal/application/shadow_ai/doc_bridge.go
Normal file
353
internal/application/shadow_ai/doc_bridge.go
Normal file
|
|
@ -0,0 +1,353 @@
|
|||
package shadow_ai
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// --- Document Review Bridge ---
|
||||
// Controlled gateway for AI access: scans documents for secrets and PII,
|
||||
// supports content redaction, and routes through the approval workflow.
|
||||
|
||||
// DocReviewStatus tracks the lifecycle of a document review.
|
||||
type DocReviewStatus string
|
||||
|
||||
const (
|
||||
DocReviewPending DocReviewStatus = "pending"
|
||||
DocReviewScanning DocReviewStatus = "scanning"
|
||||
DocReviewClean DocReviewStatus = "clean"
|
||||
DocReviewRedacted DocReviewStatus = "redacted"
|
||||
DocReviewBlocked DocReviewStatus = "blocked"
|
||||
DocReviewApproved DocReviewStatus = "approved"
|
||||
)
|
||||
|
||||
// ScanResult contains the results of scanning a document.
|
||||
type ScanResult struct {
|
||||
DocumentID string `json:"document_id"`
|
||||
Status DocReviewStatus `json:"status"`
|
||||
PIIFound []PIIMatch `json:"pii_found,omitempty"`
|
||||
SecretsFound []SecretMatch `json:"secrets_found,omitempty"`
|
||||
DataClass DataClassification `json:"data_classification"`
|
||||
ContentHash string `json:"content_hash"`
|
||||
ScannedAt time.Time `json:"scanned_at"`
|
||||
SizeBytes int `json:"size_bytes"`
|
||||
}
|
||||
|
||||
// PIIMatch represents a detected PII pattern in content.
|
||||
type PIIMatch struct {
|
||||
Type string `json:"type"` // "email", "phone", "ssn", "credit_card", "passport"
|
||||
Location int `json:"location"` // Character offset
|
||||
Length int `json:"length"`
|
||||
Masked string `json:"masked"` // Redacted value, e.g., "j***@example.com"
|
||||
}
|
||||
|
||||
// SecretMatch represents a detected secret/API key in content.
|
||||
type SecretMatch struct {
|
||||
Type string `json:"type"` // "api_key", "password", "token", "private_key"
|
||||
Location int `json:"location"`
|
||||
Length int `json:"length"`
|
||||
Provider string `json:"provider"` // "OpenAI", "AWS", "GitHub", etc.
|
||||
}
|
||||
|
||||
// DocBridge manages document scanning, redaction, and review workflow.
|
||||
type DocBridge struct {
|
||||
mu sync.RWMutex
|
||||
reviews map[string]*ScanResult
|
||||
piiPatterns []*piiPattern
|
||||
secretPats []secretPattern // Cached compiled patterns
|
||||
signatures *AISignatureDB // Reused across scans
|
||||
maxDocSize int // bytes
|
||||
}
|
||||
|
||||
type piiPattern struct {
|
||||
name string
|
||||
regex *regexp.Regexp
|
||||
maskFn func(string) string
|
||||
}
|
||||
|
||||
// NewDocBridge creates a new Document Review Bridge.
|
||||
func NewDocBridge() *DocBridge {
|
||||
return &DocBridge{
|
||||
reviews: make(map[string]*ScanResult),
|
||||
piiPatterns: defaultPIIPatterns(),
|
||||
secretPats: secretPatterns(),
|
||||
signatures: NewAISignatureDB(),
|
||||
maxDocSize: 10 * 1024 * 1024, // 10 MB
|
||||
}
|
||||
}
|
||||
|
||||
// ScanDocument scans content for PII and secrets, classifies data, returns result.
|
||||
func (db *DocBridge) ScanDocument(docID, content, userID string) *ScanResult {
|
||||
result := &ScanResult{
|
||||
DocumentID: docID,
|
||||
Status: DocReviewScanning,
|
||||
ScannedAt: time.Now(),
|
||||
SizeBytes: len(content),
|
||||
}
|
||||
|
||||
// Content hash for dedup.
|
||||
h := sha256.Sum256([]byte(content))
|
||||
result.ContentHash = fmt.Sprintf("%x", h[:])
|
||||
|
||||
// Size check.
|
||||
if len(content) > db.maxDocSize {
|
||||
result.Status = DocReviewBlocked
|
||||
result.DataClass = DataCritical
|
||||
db.store(result)
|
||||
return result
|
||||
}
|
||||
|
||||
// Scan for PII.
|
||||
result.PIIFound = db.scanPII(content)
|
||||
|
||||
// Scan for secrets (reuse cached signature DB).
|
||||
if keyType := db.signatures.ScanForAPIKeys(content); keyType != "" {
|
||||
result.SecretsFound = append(result.SecretsFound, SecretMatch{
|
||||
Type: "api_key",
|
||||
Provider: keyType,
|
||||
})
|
||||
}
|
||||
|
||||
// Scan for additional secret patterns.
|
||||
result.SecretsFound = append(result.SecretsFound, db.scanSecrets(content)...)
|
||||
|
||||
// Classify data based on findings.
|
||||
result.DataClass = db.classifyData(result)
|
||||
|
||||
// Set status based on findings.
|
||||
if len(result.SecretsFound) > 0 {
|
||||
result.Status = DocReviewBlocked
|
||||
} else if len(result.PIIFound) > 0 {
|
||||
result.Status = DocReviewRedacted
|
||||
} else {
|
||||
result.Status = DocReviewClean
|
||||
}
|
||||
|
||||
db.store(result)
|
||||
return result
|
||||
}
|
||||
|
||||
// RedactContent replaces PII and secrets in content with masked values.
|
||||
func (db *DocBridge) RedactContent(content string) string {
|
||||
for _, p := range db.piiPatterns {
|
||||
content = p.regex.ReplaceAllStringFunc(content, p.maskFn)
|
||||
}
|
||||
|
||||
// Redact common secret patterns (cached).
|
||||
for _, sp := range db.secretPats {
|
||||
content = sp.regex.ReplaceAllString(content, sp.replacement)
|
||||
}
|
||||
|
||||
return content
|
||||
}
|
||||
|
||||
// GetReview returns a scan result by document ID.
|
||||
func (db *DocBridge) GetReview(docID string) (*ScanResult, bool) {
|
||||
db.mu.RLock()
|
||||
defer db.mu.RUnlock()
|
||||
r, ok := db.reviews[docID]
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
cp := *r
|
||||
return &cp, true
|
||||
}
|
||||
|
||||
// RecentReviews returns the N most recent reviews.
|
||||
func (db *DocBridge) RecentReviews(limit int) []ScanResult {
|
||||
db.mu.RLock()
|
||||
defer db.mu.RUnlock()
|
||||
|
||||
results := make([]ScanResult, 0, len(db.reviews))
|
||||
for _, r := range db.reviews {
|
||||
results = append(results, *r)
|
||||
}
|
||||
|
||||
// Sort by time desc (simple bubble for bounded set).
|
||||
for i := 0; i < len(results); i++ {
|
||||
for j := i + 1; j < len(results); j++ {
|
||||
if results[j].ScannedAt.After(results[i].ScannedAt) {
|
||||
results[i], results[j] = results[j], results[i]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(results) > limit {
|
||||
results = results[:limit]
|
||||
}
|
||||
return results
|
||||
}
|
||||
|
||||
// Stats returns aggregate document review statistics.
|
||||
func (db *DocBridge) Stats() map[string]int {
|
||||
db.mu.RLock()
|
||||
defer db.mu.RUnlock()
|
||||
|
||||
stats := map[string]int{
|
||||
"total": len(db.reviews),
|
||||
"clean": 0,
|
||||
"redacted": 0,
|
||||
"blocked": 0,
|
||||
}
|
||||
for _, r := range db.reviews {
|
||||
switch r.Status {
|
||||
case DocReviewClean:
|
||||
stats["clean"]++
|
||||
case DocReviewRedacted:
|
||||
stats["redacted"]++
|
||||
case DocReviewBlocked:
|
||||
stats["blocked"]++
|
||||
}
|
||||
}
|
||||
return stats
|
||||
}
|
||||
|
||||
func (db *DocBridge) store(result *ScanResult) {
|
||||
db.mu.Lock()
|
||||
defer db.mu.Unlock()
|
||||
db.reviews[result.DocumentID] = result
|
||||
}
|
||||
|
||||
// scanPII runs all PII patterns against content.
|
||||
func (db *DocBridge) scanPII(content string) []PIIMatch {
|
||||
var matches []PIIMatch
|
||||
for _, p := range db.piiPatterns {
|
||||
locs := p.regex.FindAllStringIndex(content, -1)
|
||||
for _, loc := range locs {
|
||||
matched := content[loc[0]:loc[1]]
|
||||
matches = append(matches, PIIMatch{
|
||||
Type: p.name,
|
||||
Location: loc[0],
|
||||
Length: loc[1] - loc[0],
|
||||
Masked: p.maskFn(matched),
|
||||
})
|
||||
}
|
||||
}
|
||||
return matches
|
||||
}
|
||||
|
||||
// scanSecrets scans for common secret patterns beyond AI API keys.
|
||||
func (db *DocBridge) scanSecrets(content string) []SecretMatch {
|
||||
var matches []SecretMatch
|
||||
for _, sp := range db.secretPats {
|
||||
locs := sp.regex.FindAllStringIndex(content, -1)
|
||||
for _, loc := range locs {
|
||||
matches = append(matches, SecretMatch{
|
||||
Type: sp.secretType,
|
||||
Location: loc[0],
|
||||
Length: loc[1] - loc[0],
|
||||
Provider: sp.provider,
|
||||
})
|
||||
}
|
||||
}
|
||||
return matches
|
||||
}
|
||||
|
||||
// classifyData determines the data classification level based on scan results.
|
||||
func (db *DocBridge) classifyData(result *ScanResult) DataClassification {
|
||||
if len(result.SecretsFound) > 0 {
|
||||
return DataCritical
|
||||
}
|
||||
|
||||
hasSensitivePII := false
|
||||
for _, pii := range result.PIIFound {
|
||||
switch pii.Type {
|
||||
case "ssn", "credit_card", "passport":
|
||||
return DataCritical
|
||||
case "email", "phone":
|
||||
hasSensitivePII = true
|
||||
}
|
||||
}
|
||||
|
||||
if hasSensitivePII {
|
||||
return DataConfidential
|
||||
}
|
||||
|
||||
if result.SizeBytes > 1024*1024 { // >1MB
|
||||
return DataInternal
|
||||
}
|
||||
|
||||
return DataPublic
|
||||
}
|
||||
|
||||
// --- PII Patterns ---
|
||||
|
||||
func defaultPIIPatterns() []*piiPattern {
|
||||
return []*piiPattern{
|
||||
{
|
||||
name: "email",
|
||||
regex: regexp.MustCompile(`[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}`),
|
||||
maskFn: func(s string) string {
|
||||
parts := strings.SplitN(s, "@", 2)
|
||||
if len(parts) != 2 {
|
||||
return "***@***"
|
||||
}
|
||||
if len(parts[0]) <= 1 {
|
||||
return "*@" + parts[1]
|
||||
}
|
||||
return string(parts[0][0]) + "***@" + parts[1]
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "phone",
|
||||
regex: regexp.MustCompile(`\+?[1-9]\d{0,2}[\s\-]?\(?\d{3}\)?[\s\-]?\d{3}[\s\-]?\d{2,4}`),
|
||||
maskFn: func(s string) string {
|
||||
if len(s) < 4 {
|
||||
return "***"
|
||||
}
|
||||
return s[:2] + strings.Repeat("*", len(s)-4) + s[len(s)-2:]
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "ssn",
|
||||
regex: regexp.MustCompile(`\b\d{3}-\d{2}-\d{4}\b`),
|
||||
maskFn: func(_ string) string {
|
||||
return "***-**-****"
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "credit_card",
|
||||
regex: regexp.MustCompile(`\b(?:\d{4}[\s\-]?){3}\d{4}\b`),
|
||||
maskFn: func(s string) string {
|
||||
clean := strings.ReplaceAll(strings.ReplaceAll(s, "-", ""), " ", "")
|
||||
if len(clean) < 4 {
|
||||
return "****"
|
||||
}
|
||||
return strings.Repeat("*", len(clean)-4) + clean[len(clean)-4:]
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "passport",
|
||||
regex: regexp.MustCompile(`\b[A-Z]{1,2}\d{6,9}\b`),
|
||||
maskFn: func(s string) string {
|
||||
if len(s) <= 2 {
|
||||
return "**"
|
||||
}
|
||||
return s[:2] + strings.Repeat("*", len(s)-2)
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
type secretPattern struct {
|
||||
secretType string
|
||||
provider string
|
||||
regex *regexp.Regexp
|
||||
replacement string
|
||||
}
|
||||
|
||||
func secretPatterns() []secretPattern {
|
||||
return []secretPattern{
|
||||
{secretType: "aws_key", provider: "AWS", regex: regexp.MustCompile(`AKIA[0-9A-Z]{16}`), replacement: "[AWS_KEY_REDACTED]"},
|
||||
{secretType: "github_token", provider: "GitHub", regex: regexp.MustCompile(`ghp_[a-zA-Z0-9]{36}`), replacement: "[GITHUB_TOKEN_REDACTED]"},
|
||||
{secretType: "github_token", provider: "GitHub", regex: regexp.MustCompile(`github_pat_[a-zA-Z0-9_]{82}`), replacement: "[GITHUB_PAT_REDACTED]"},
|
||||
{secretType: "slack_token", provider: "Slack", regex: regexp.MustCompile(`xoxb-[0-9]{10,13}-[0-9]{10,13}-[a-zA-Z0-9]{24}`), replacement: "[SLACK_TOKEN_REDACTED]"},
|
||||
{secretType: "private_key", provider: "Generic", regex: regexp.MustCompile(`-----BEGIN (?:RSA |EC |DSA )?PRIVATE KEY-----`), replacement: "[PRIVATE_KEY_REDACTED]"},
|
||||
{secretType: "password", provider: "Generic", regex: regexp.MustCompile(`(?i)password\s*[=:]\s*['"]?[^\s'"]{8,}`), replacement: "[PASSWORD_REDACTED]"},
|
||||
{secretType: "connection_string", provider: "Database", regex: regexp.MustCompile(`(?i)(?:mysql|postgres|mongodb)://[^\s]+`), replacement: "[DB_CONN_REDACTED]"},
|
||||
}
|
||||
}
|
||||
148
internal/application/shadow_ai/fallback.go
Normal file
148
internal/application/shadow_ai/fallback.go
Normal file
|
|
@ -0,0 +1,148 @@
|
|||
package shadow_ai
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"time"
|
||||
)
|
||||
|
||||
// FallbackManager provides priority-based enforcement with graceful degradation.
|
||||
// Tries enforcement points in priority order; falls back to detect_only if all are offline.
|
||||
type FallbackManager struct {
|
||||
registry *PluginRegistry
|
||||
priority []PluginType // e.g., ["proxy", "firewall", "edr"]
|
||||
strategy string // "detect_only" | "alert_only"
|
||||
logger *slog.Logger
|
||||
|
||||
// Event logging for detect-only fallback.
|
||||
eventLogFn func(event ShadowAIEvent)
|
||||
}
|
||||
|
||||
// NewFallbackManager creates a new fallback manager with the given enforcement priority.
|
||||
func NewFallbackManager(registry *PluginRegistry, strategy string) *FallbackManager {
|
||||
if strategy == "" {
|
||||
strategy = "detect_only"
|
||||
}
|
||||
return &FallbackManager{
|
||||
registry: registry,
|
||||
priority: []PluginType{PluginTypeProxy, PluginTypeFirewall, PluginTypeEDR},
|
||||
strategy: strategy,
|
||||
logger: slog.Default().With("component", "shadow-ai-fallback"),
|
||||
}
|
||||
}
|
||||
|
||||
// SetEventLogger sets the callback for logging detection-only events.
|
||||
func (fm *FallbackManager) SetEventLogger(fn func(ShadowAIEvent)) {
|
||||
fm.eventLogFn = fn
|
||||
}
|
||||
|
||||
// BlockDomain attempts to block a domain using the highest-priority healthy plugin.
|
||||
// Returns the vendor that enforced, or falls back to detect_only mode.
|
||||
func (fm *FallbackManager) BlockDomain(ctx context.Context, domain, reason string) (enforcedBy string, err error) {
|
||||
for _, pType := range fm.priority {
|
||||
plugins := fm.registry.GetByType(pType)
|
||||
for _, plugin := range plugins {
|
||||
ne, ok := plugin.(NetworkEnforcer)
|
||||
if !ok {
|
||||
// Try WebGateway for URL-based blocking.
|
||||
if wg, ok := plugin.(WebGateway); ok {
|
||||
vendor := wg.Vendor()
|
||||
if !fm.registry.IsHealthy(vendor) {
|
||||
continue
|
||||
}
|
||||
if err := wg.BlockURL(ctx, domain, reason); err != nil {
|
||||
fm.logger.Warn("block failed on gateway", "vendor", vendor, "error", err)
|
||||
continue
|
||||
}
|
||||
return vendor, nil
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
vendor := ne.Vendor()
|
||||
if !fm.registry.IsHealthy(vendor) {
|
||||
continue
|
||||
}
|
||||
if err := ne.BlockDomain(ctx, domain, reason); err != nil {
|
||||
fm.logger.Warn("block failed on enforcer", "vendor", vendor, "error", err)
|
||||
continue
|
||||
}
|
||||
return vendor, nil
|
||||
}
|
||||
}
|
||||
|
||||
// All enforcement points unavailable — fallback.
|
||||
fm.logger.Warn("all enforcement points unavailable, falling to detect_only",
|
||||
"domain", domain,
|
||||
"strategy", fm.strategy,
|
||||
)
|
||||
fm.logDetectOnly(domain, reason)
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// BlockIP attempts to block an IP using the highest-priority healthy firewall.
|
||||
func (fm *FallbackManager) BlockIP(ctx context.Context, ip string, duration time.Duration, reason string) (enforcedBy string, err error) {
|
||||
enforcers := fm.registry.GetNetworkEnforcers()
|
||||
for _, ne := range enforcers {
|
||||
vendor := ne.Vendor()
|
||||
if !fm.registry.IsHealthy(vendor) {
|
||||
continue
|
||||
}
|
||||
if err := ne.BlockIP(ctx, ip, duration, reason); err != nil {
|
||||
fm.logger.Warn("block IP failed", "vendor", vendor, "error", err)
|
||||
continue
|
||||
}
|
||||
return vendor, nil
|
||||
}
|
||||
|
||||
fm.logger.Warn("no healthy enforcer for IP block, falling to detect_only",
|
||||
"ip", ip,
|
||||
"strategy", fm.strategy,
|
||||
)
|
||||
fm.logDetectOnly(ip, reason)
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// IsolateHost attempts to isolate a host using the highest-priority healthy EDR.
|
||||
func (fm *FallbackManager) IsolateHost(ctx context.Context, hostname string) (enforcedBy string, err error) {
|
||||
controllers := fm.registry.GetEndpointControllers()
|
||||
for _, ec := range controllers {
|
||||
vendor := ec.Vendor()
|
||||
if !fm.registry.IsHealthy(vendor) {
|
||||
continue
|
||||
}
|
||||
if err := ec.IsolateHost(ctx, hostname); err != nil {
|
||||
fm.logger.Warn("isolate failed", "vendor", vendor, "error", err)
|
||||
continue
|
||||
}
|
||||
return vendor, nil
|
||||
}
|
||||
|
||||
fm.logger.Warn("no healthy EDR for host isolation, falling to detect_only",
|
||||
"hostname", hostname,
|
||||
"strategy", fm.strategy,
|
||||
)
|
||||
return "", fmt.Errorf("no healthy EDR available for host isolation")
|
||||
}
|
||||
|
||||
// logDetectOnly records a detection-only event when no enforcement is possible.
|
||||
func (fm *FallbackManager) logDetectOnly(target, reason string) {
|
||||
if fm.eventLogFn != nil {
|
||||
fm.eventLogFn(ShadowAIEvent{
|
||||
Destination: target,
|
||||
DetectionMethod: DetectNetwork,
|
||||
Action: "detect_only",
|
||||
Metadata: map[string]string{
|
||||
"reason": reason,
|
||||
"fallback_strategy": fm.strategy,
|
||||
},
|
||||
Timestamp: time.Now(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Strategy returns the configured fallback strategy.
|
||||
func (fm *FallbackManager) Strategy() string {
|
||||
return fm.strategy
|
||||
}
|
||||
163
internal/application/shadow_ai/health.go
Normal file
163
internal/application/shadow_ai/health.go
Normal file
|
|
@ -0,0 +1,163 @@
|
|||
package shadow_ai
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// PluginStatus represents a plugin's operational state.
|
||||
type PluginStatus string
|
||||
|
||||
const (
|
||||
PluginStatusHealthy PluginStatus = "healthy"
|
||||
PluginStatusDegraded PluginStatus = "degraded"
|
||||
PluginStatusOffline PluginStatus = "offline"
|
||||
)
|
||||
|
||||
// PluginHealth tracks the health state of a single plugin.
|
||||
type PluginHealth struct {
|
||||
Vendor string `json:"vendor"`
|
||||
Type PluginType `json:"type"`
|
||||
Status PluginStatus `json:"status"`
|
||||
LastCheck time.Time `json:"last_check"`
|
||||
Consecutive int `json:"consecutive_failures"`
|
||||
Latency time.Duration `json:"latency"`
|
||||
LastError string `json:"last_error,omitempty"`
|
||||
}
|
||||
|
||||
// MaxConsecutivePluginFailures before marking offline.
|
||||
const MaxConsecutivePluginFailures = 3
|
||||
|
||||
// HealthChecker performs continuous health monitoring of all registered plugins.
|
||||
type HealthChecker struct {
|
||||
mu sync.RWMutex
|
||||
registry *PluginRegistry
|
||||
interval time.Duration
|
||||
alertFn func(vendor string, status PluginStatus, msg string)
|
||||
logger *slog.Logger
|
||||
}
|
||||
|
||||
// NewHealthChecker creates a health checker that monitors plugin health.
|
||||
func NewHealthChecker(registry *PluginRegistry, interval time.Duration, alertFn func(string, PluginStatus, string)) *HealthChecker {
|
||||
if interval <= 0 {
|
||||
interval = 30 * time.Second
|
||||
}
|
||||
return &HealthChecker{
|
||||
registry: registry,
|
||||
interval: interval,
|
||||
alertFn: alertFn,
|
||||
logger: slog.Default().With("component", "shadow-ai-health"),
|
||||
}
|
||||
}
|
||||
|
||||
// Start begins continuous health monitoring. Blocks until ctx is cancelled.
|
||||
func (hc *HealthChecker) Start(ctx context.Context) {
|
||||
hc.logger.Info("health checker started", "interval", hc.interval)
|
||||
ticker := time.NewTicker(hc.interval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
hc.logger.Info("health checker stopped")
|
||||
return
|
||||
case <-ticker.C:
|
||||
hc.checkAllPlugins(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// checkAllPlugins runs health checks on all registered plugins.
|
||||
func (hc *HealthChecker) checkAllPlugins(ctx context.Context) {
|
||||
vendors := hc.registry.Vendors()
|
||||
|
||||
for _, vendor := range vendors {
|
||||
plugin, ok := hc.registry.Get(vendor)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
existing, _ := hc.registry.GetHealth(vendor)
|
||||
if existing == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
err := hc.checkPlugin(ctx, plugin)
|
||||
latency := time.Since(start)
|
||||
|
||||
health := &PluginHealth{
|
||||
Vendor: vendor,
|
||||
Type: existing.Type,
|
||||
LastCheck: time.Now(),
|
||||
Latency: latency,
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
health.Consecutive = existing.Consecutive + 1
|
||||
health.LastError = err.Error()
|
||||
|
||||
if health.Consecutive >= MaxConsecutivePluginFailures {
|
||||
health.Status = PluginStatusOffline
|
||||
if existing.Status != PluginStatusOffline {
|
||||
hc.logger.Error("plugin went OFFLINE",
|
||||
"vendor", vendor,
|
||||
"consecutive", health.Consecutive,
|
||||
"error", err,
|
||||
)
|
||||
if hc.alertFn != nil {
|
||||
hc.alertFn(vendor, PluginStatusOffline,
|
||||
fmt.Sprintf("Plugin %s offline after %d consecutive failures: %v",
|
||||
vendor, health.Consecutive, err))
|
||||
}
|
||||
}
|
||||
} else {
|
||||
health.Status = PluginStatusDegraded
|
||||
hc.logger.Warn("plugin health check failed",
|
||||
"vendor", vendor,
|
||||
"consecutive", health.Consecutive,
|
||||
"error", err,
|
||||
)
|
||||
}
|
||||
} else {
|
||||
health.Status = PluginStatusHealthy
|
||||
health.Consecutive = 0
|
||||
|
||||
// Log recovery if previously degraded/offline.
|
||||
if existing.Status != PluginStatusHealthy {
|
||||
hc.logger.Info("plugin recovered", "vendor", vendor, "latency", latency)
|
||||
if hc.alertFn != nil {
|
||||
hc.alertFn(vendor, PluginStatusHealthy,
|
||||
fmt.Sprintf("Plugin %s recovered, latency %s", vendor, latency))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
hc.registry.SetHealth(vendor, health)
|
||||
}
|
||||
}
|
||||
|
||||
// checkPlugin runs the health check for a single plugin.
|
||||
func (hc *HealthChecker) checkPlugin(ctx context.Context, plugin interface{}) error {
|
||||
checkCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
switch p := plugin.(type) {
|
||||
case NetworkEnforcer:
|
||||
return p.HealthCheck(checkCtx)
|
||||
case EndpointController:
|
||||
return p.HealthCheck(checkCtx)
|
||||
case WebGateway:
|
||||
return p.HealthCheck(checkCtx)
|
||||
default:
|
||||
return fmt.Errorf("plugin does not implement HealthCheck")
|
||||
}
|
||||
}
|
||||
|
||||
// CheckNow runs an immediate health check on all plugins (non-blocking).
|
||||
func (hc *HealthChecker) CheckNow(ctx context.Context) {
|
||||
hc.checkAllPlugins(ctx)
|
||||
}
|
||||
225
internal/application/shadow_ai/interfaces.go
Normal file
225
internal/application/shadow_ai/interfaces.go
Normal file
|
|
@ -0,0 +1,225 @@
|
|||
// Package shadow_ai implements the Sentinel Shadow AI Control Module.
|
||||
//
|
||||
// Five levels of shadow AI management:
|
||||
//
|
||||
// L1 — Universal Integration Layer: plugin-based enforcement (firewall, EDR, proxy)
|
||||
// L2 — Detection Engine: network signatures, endpoint, API keys, behavioral
|
||||
// L3 — Document Review Bridge: controlled LLM access with PII/secret scanning
|
||||
// L4 — Approval Workflow: tiered data classification and manager/SOC approval
|
||||
// L5 — SOC Integration: dashboard, correlation rules, playbooks, compliance
|
||||
package shadow_ai
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
)
|
||||
|
||||
// --- Plugin Interfaces ---
|
||||
|
||||
// NetworkEnforcer is the universal interface for ALL firewalls.
|
||||
// Implementations: Check Point, Cisco ASA/FMC, Palo Alto, Fortinet.
|
||||
type NetworkEnforcer interface {
|
||||
// BlockIP blocks an IP address for the given duration.
|
||||
BlockIP(ctx context.Context, ip string, duration time.Duration, reason string) error
|
||||
|
||||
// BlockDomain blocks a domain name.
|
||||
BlockDomain(ctx context.Context, domain string, reason string) error
|
||||
|
||||
// UnblockIP removes an IP block.
|
||||
UnblockIP(ctx context.Context, ip string) error
|
||||
|
||||
// UnblockDomain removes a domain block.
|
||||
UnblockDomain(ctx context.Context, domain string) error
|
||||
|
||||
// HealthCheck verifies the firewall API is reachable.
|
||||
HealthCheck(ctx context.Context) error
|
||||
|
||||
// Vendor returns the vendor identifier (e.g., "checkpoint", "cisco", "paloalto").
|
||||
Vendor() string
|
||||
}
|
||||
|
||||
// EndpointController is the universal interface for ALL EDR systems.
|
||||
// Implementations: CrowdStrike, SentinelOne, Microsoft Defender.
|
||||
type EndpointController interface {
|
||||
// IsolateHost quarantines a host from the network.
|
||||
IsolateHost(ctx context.Context, hostname string) error
|
||||
|
||||
// ReleaseHost removes host isolation.
|
||||
ReleaseHost(ctx context.Context, hostname string) error
|
||||
|
||||
// KillProcess terminates a process on a remote host.
|
||||
KillProcess(ctx context.Context, hostname string, pid int) error
|
||||
|
||||
// QuarantineFile moves a file to quarantine on a remote host.
|
||||
QuarantineFile(ctx context.Context, hostname string, path string) error
|
||||
|
||||
// HealthCheck verifies the EDR API is reachable.
|
||||
HealthCheck(ctx context.Context) error
|
||||
|
||||
// Vendor returns the vendor identifier (e.g., "crowdstrike", "sentinelone", "defender").
|
||||
Vendor() string
|
||||
}
|
||||
|
||||
// WebGateway is the universal interface for ALL proxy/CASB systems.
|
||||
// Implementations: Zscaler, Netskope, Squid, BlueCoat.
|
||||
type WebGateway interface {
|
||||
// BlockURL adds a URL to the blocklist.
|
||||
BlockURL(ctx context.Context, url string, reason string) error
|
||||
|
||||
// UnblockURL removes a URL from the blocklist.
|
||||
UnblockURL(ctx context.Context, url string) error
|
||||
|
||||
// BlockCategory blocks an entire URL category (e.g., "Artificial Intelligence").
|
||||
BlockCategory(ctx context.Context, category string) error
|
||||
|
||||
// HealthCheck verifies the gateway API is reachable.
|
||||
HealthCheck(ctx context.Context) error
|
||||
|
||||
// Vendor returns the vendor identifier (e.g., "zscaler", "netskope", "squid").
|
||||
Vendor() string
|
||||
}
|
||||
|
||||
// Initializer is implemented by plugins that need configuration before use.
|
||||
type Initializer interface {
|
||||
Initialize(config map[string]interface{}) error
|
||||
}
|
||||
|
||||
// --- Plugin Configuration ---
|
||||
|
||||
// PluginType categorizes enforcement points.
|
||||
type PluginType string
|
||||
|
||||
const (
|
||||
PluginTypeFirewall PluginType = "firewall"
|
||||
PluginTypeEDR PluginType = "edr"
|
||||
PluginTypeProxy PluginType = "proxy"
|
||||
PluginTypeDNS PluginType = "dns"
|
||||
)
|
||||
|
||||
// PluginConfig defines a vendor plugin configuration loaded from YAML.
|
||||
type PluginConfig struct {
|
||||
Type PluginType `yaml:"type" json:"type"`
|
||||
Vendor string `yaml:"vendor" json:"vendor"`
|
||||
Enabled bool `yaml:"enabled" json:"enabled"`
|
||||
Config map[string]interface{} `yaml:"config" json:"config"`
|
||||
}
|
||||
|
||||
// IntegrationConfig is the top-level Shadow AI configuration.
|
||||
type IntegrationConfig struct {
|
||||
Plugins []PluginConfig `yaml:"plugins" json:"plugins"`
|
||||
FallbackStrategy string `yaml:"fallback_strategy" json:"fallback_strategy"` // "detect_only" | "alert_only"
|
||||
HealthCheckInterval time.Duration `yaml:"health_check_interval" json:"health_check_interval"` // default: 30s
|
||||
}
|
||||
|
||||
// --- Domain Types ---
|
||||
|
||||
// DetectionMethod identifies how a shadow AI usage was detected.
|
||||
type DetectionMethod string
|
||||
|
||||
const (
|
||||
DetectNetwork DetectionMethod = "network" // Domain/IP match
|
||||
DetectHTTP DetectionMethod = "http" // HTTP header signature
|
||||
DetectTLS DetectionMethod = "tls" // TLS/JA3 fingerprint
|
||||
DetectProcess DetectionMethod = "process" // AI tool process execution
|
||||
DetectAPIKey DetectionMethod = "api_key" // AI API key in payload
|
||||
DetectBehavioral DetectionMethod = "behavioral" // Anomalous AI access pattern
|
||||
DetectClipboard DetectionMethod = "clipboard" // Large clipboard → AI browser pattern
|
||||
)
|
||||
|
||||
// DataClassification determines the approval tier required.
|
||||
type DataClassification string
|
||||
|
||||
const (
|
||||
DataPublic DataClassification = "PUBLIC"
|
||||
DataInternal DataClassification = "INTERNAL"
|
||||
DataConfidential DataClassification = "CONFIDENTIAL"
|
||||
DataCritical DataClassification = "CRITICAL"
|
||||
)
|
||||
|
||||
// ShadowAIEvent is a detected shadow AI usage attempt.
|
||||
type ShadowAIEvent struct {
|
||||
ID string `json:"id"`
|
||||
UserID string `json:"user_id"`
|
||||
Hostname string `json:"hostname"`
|
||||
Destination string `json:"destination"` // Target AI service domain/IP
|
||||
AIService string `json:"ai_service"` // "chatgpt", "claude", "gemini", etc.
|
||||
DetectionMethod DetectionMethod `json:"detection_method"`
|
||||
Action string `json:"action"` // "blocked", "allowed", "pending"
|
||||
EnforcedBy string `json:"enforced_by"` // Plugin vendor that enforced
|
||||
DataSize int64 `json:"data_size"` // Bytes sent to AI
|
||||
Timestamp time.Time `json:"timestamp"`
|
||||
Metadata map[string]string `json:"metadata,omitempty"`
|
||||
}
|
||||
|
||||
// AIServiceInfo describes a known AI service for signature matching.
|
||||
type AIServiceInfo struct {
|
||||
Name string `json:"name"` // "ChatGPT", "Claude", "Gemini"
|
||||
Vendor string `json:"vendor"` // "OpenAI", "Anthropic", "Google"
|
||||
Domains []string `json:"domains"` // ["*.openai.com", "chat.openai.com"]
|
||||
Category string `json:"category"` // "llm", "image_gen", "code_assist"
|
||||
}
|
||||
|
||||
// BlockRequest is an API request to manually block a target.
|
||||
type BlockRequest struct {
|
||||
TargetType string `json:"target_type"` // "ip", "domain", "user"
|
||||
Target string `json:"target"`
|
||||
Duration time.Duration `json:"duration"`
|
||||
Reason string `json:"reason"`
|
||||
BlockedBy string `json:"blocked_by"` // RBAC user
|
||||
}
|
||||
|
||||
// ShadowAIStats provides aggregate statistics for the dashboard.
|
||||
type ShadowAIStats struct {
|
||||
TimeRange string `json:"time_range"` // "24h", "7d", "30d"
|
||||
Total int `json:"total_attempts"`
|
||||
Blocked int `json:"blocked"`
|
||||
Approved int `json:"approved"`
|
||||
Pending int `json:"pending"`
|
||||
ByService map[string]int `json:"by_service"`
|
||||
ByDepartment map[string]int `json:"by_department"`
|
||||
TopViolators []Violator `json:"top_violators"`
|
||||
}
|
||||
|
||||
// Violator tracks a user's shadow AI violation count.
|
||||
type Violator struct {
|
||||
UserID string `json:"user_id"`
|
||||
Attempts int `json:"attempts"`
|
||||
}
|
||||
|
||||
// ApprovalTier defines the approval requirements for a data classification level.
|
||||
type ApprovalTier struct {
|
||||
Name string `yaml:"name" json:"name"`
|
||||
DataClass DataClassification `yaml:"data_class" json:"data_class"`
|
||||
ApprovalNeeded []string `yaml:"approval_needed" json:"approval_needed"` // ["manager"], ["manager", "soc"], ["ciso"]
|
||||
SLA time.Duration `yaml:"sla" json:"sla"`
|
||||
AutoApprove bool `yaml:"auto_approve" json:"auto_approve"`
|
||||
}
|
||||
|
||||
// ApprovalRequest tracks a pending approval for AI access.
|
||||
type ApprovalRequest struct {
|
||||
ID string `json:"id"`
|
||||
DocID string `json:"doc_id"`
|
||||
UserID string `json:"user_id"`
|
||||
Tier string `json:"tier"`
|
||||
DataClass DataClassification `json:"data_class"`
|
||||
Status string `json:"status"` // "pending", "approved", "denied", "expired"
|
||||
ApprovedBy string `json:"approved_by,omitempty"`
|
||||
DeniedBy string `json:"denied_by,omitempty"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
ExpiresAt time.Time `json:"expires_at"`
|
||||
ResolvedAt time.Time `json:"resolved_at,omitempty"`
|
||||
}
|
||||
|
||||
// ComplianceReport is the Shadow AI compliance report for GDPR/SOC2/EU AI Act.
|
||||
type ComplianceReport struct {
|
||||
GeneratedAt time.Time `json:"generated_at"`
|
||||
Period string `json:"period"` // "monthly", "quarterly"
|
||||
TotalInteractions int `json:"total_interactions"`
|
||||
BlockedAttempts int `json:"blocked_attempts"`
|
||||
ApprovedReviews int `json:"approved_reviews"`
|
||||
PIIDetected int `json:"pii_detected"`
|
||||
SecretsDetected int `json:"secrets_detected"`
|
||||
AuditComplete bool `json:"audit_complete"`
|
||||
Regulations []string `json:"regulations"` // ["GDPR", "SOC2", "EU AI Act"]
|
||||
}
|
||||
212
internal/application/shadow_ai/plugins.go
Normal file
212
internal/application/shadow_ai/plugins.go
Normal file
|
|
@ -0,0 +1,212 @@
|
|||
package shadow_ai
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"time"
|
||||
)
|
||||
|
||||
// --- Vendor Plugin Stubs ---
|
||||
// Reference implementations for major security vendors.
|
||||
// These stubs implement the full interface with logging but no real API calls.
|
||||
// Production deployments replace these with real vendor SDK integrations.
|
||||
|
||||
// CheckPointEnforcer is a stub implementation for Check Point firewalls.
|
||||
type CheckPointEnforcer struct {
|
||||
apiURL string
|
||||
apiKey string
|
||||
logger *slog.Logger
|
||||
}
|
||||
|
||||
func NewCheckPointEnforcer() *CheckPointEnforcer {
|
||||
return &CheckPointEnforcer{
|
||||
logger: slog.Default().With("component", "shadow-ai-plugin-checkpoint"),
|
||||
}
|
||||
}
|
||||
|
||||
func (c *CheckPointEnforcer) Initialize(config map[string]interface{}) error {
|
||||
if url, ok := config["api_url"].(string); ok {
|
||||
c.apiURL = url
|
||||
}
|
||||
if key, ok := config["api_key"].(string); ok {
|
||||
c.apiKey = key
|
||||
}
|
||||
if c.apiURL == "" {
|
||||
return fmt.Errorf("checkpoint: api_url required")
|
||||
}
|
||||
c.logger.Info("initialized", "api_url", c.apiURL)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *CheckPointEnforcer) BlockIP(_ context.Context, ip string, duration time.Duration, reason string) error {
|
||||
c.logger.Info("block IP", "ip", ip, "duration", duration, "reason", reason)
|
||||
// Stub: would call Check Point Management API POST /web_api/add-host
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *CheckPointEnforcer) BlockDomain(_ context.Context, domain string, reason string) error {
|
||||
c.logger.Info("block domain", "domain", domain, "reason", reason)
|
||||
// Stub: would create application-site-category block rule
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *CheckPointEnforcer) UnblockIP(_ context.Context, ip string) error {
|
||||
c.logger.Info("unblock IP", "ip", ip)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *CheckPointEnforcer) UnblockDomain(_ context.Context, domain string) error {
|
||||
c.logger.Info("unblock domain", "domain", domain)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *CheckPointEnforcer) HealthCheck(ctx context.Context) error {
|
||||
if c.apiURL == "" {
|
||||
return fmt.Errorf("not configured")
|
||||
}
|
||||
// Stub: would call GET /web_api/show-session
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *CheckPointEnforcer) Vendor() string { return "checkpoint" }
|
||||
|
||||
// CrowdStrikeController is a stub implementation for CrowdStrike Falcon EDR.
|
||||
type CrowdStrikeController struct {
|
||||
clientID string
|
||||
clientSecret string
|
||||
baseURL string
|
||||
logger *slog.Logger
|
||||
}
|
||||
|
||||
func NewCrowdStrikeController() *CrowdStrikeController {
|
||||
return &CrowdStrikeController{
|
||||
baseURL: "https://api.crowdstrike.com",
|
||||
logger: slog.Default().With("component", "shadow-ai-plugin-crowdstrike"),
|
||||
}
|
||||
}
|
||||
|
||||
func (cs *CrowdStrikeController) Initialize(config map[string]interface{}) error {
|
||||
if id, ok := config["client_id"].(string); ok {
|
||||
cs.clientID = id
|
||||
}
|
||||
if secret, ok := config["client_secret"].(string); ok {
|
||||
cs.clientSecret = secret
|
||||
}
|
||||
if url, ok := config["base_url"].(string); ok {
|
||||
cs.baseURL = url
|
||||
}
|
||||
if cs.clientID == "" {
|
||||
return fmt.Errorf("crowdstrike: client_id required")
|
||||
}
|
||||
cs.logger.Info("initialized", "base_url", cs.baseURL)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (cs *CrowdStrikeController) IsolateHost(_ context.Context, hostname string) error {
|
||||
cs.logger.Info("isolate host", "hostname", hostname)
|
||||
// Stub: would call POST /devices/entities/devices-actions/v2?action_name=contain
|
||||
return nil
|
||||
}
|
||||
|
||||
func (cs *CrowdStrikeController) ReleaseHost(_ context.Context, hostname string) error {
|
||||
cs.logger.Info("release host", "hostname", hostname)
|
||||
// Stub: would call POST /devices/entities/devices-actions/v2?action_name=lift_containment
|
||||
return nil
|
||||
}
|
||||
|
||||
func (cs *CrowdStrikeController) KillProcess(_ context.Context, hostname string, pid int) error {
|
||||
cs.logger.Info("kill process", "hostname", hostname, "pid", pid)
|
||||
// Stub: would use RTR session to kill process
|
||||
return nil
|
||||
}
|
||||
|
||||
func (cs *CrowdStrikeController) QuarantineFile(_ context.Context, hostname, path string) error {
|
||||
cs.logger.Info("quarantine file", "hostname", hostname, "path", path)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (cs *CrowdStrikeController) HealthCheck(ctx context.Context) error {
|
||||
if cs.clientID == "" {
|
||||
return fmt.Errorf("not configured")
|
||||
}
|
||||
// Stub: would call GET /sensors/queries/sensors/v1?limit=1
|
||||
return nil
|
||||
}
|
||||
|
||||
func (cs *CrowdStrikeController) Vendor() string { return "crowdstrike" }
|
||||
|
||||
// ZscalerGateway is a stub implementation for Zscaler Internet Access.
|
||||
type ZscalerGateway struct {
|
||||
cloudName string
|
||||
apiKey string
|
||||
username string
|
||||
password string
|
||||
logger *slog.Logger
|
||||
}
|
||||
|
||||
func NewZscalerGateway() *ZscalerGateway {
|
||||
return &ZscalerGateway{
|
||||
logger: slog.Default().With("component", "shadow-ai-plugin-zscaler"),
|
||||
}
|
||||
}
|
||||
|
||||
func (z *ZscalerGateway) Initialize(config map[string]interface{}) error {
|
||||
if cloud, ok := config["cloud_name"].(string); ok {
|
||||
z.cloudName = cloud
|
||||
}
|
||||
if key, ok := config["api_key"].(string); ok {
|
||||
z.apiKey = key
|
||||
}
|
||||
if user, ok := config["username"].(string); ok {
|
||||
z.username = user
|
||||
}
|
||||
if pass, ok := config["password"].(string); ok {
|
||||
z.password = pass
|
||||
}
|
||||
if z.cloudName == "" {
|
||||
return fmt.Errorf("zscaler: cloud_name required")
|
||||
}
|
||||
z.logger.Info("initialized", "cloud", z.cloudName)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (z *ZscalerGateway) BlockURL(_ context.Context, url, reason string) error {
|
||||
z.logger.Info("block URL", "url", url, "reason", reason)
|
||||
// Stub: would call PUT /webApplicationRules to add URL to block list
|
||||
return nil
|
||||
}
|
||||
|
||||
func (z *ZscalerGateway) UnblockURL(_ context.Context, url string) error {
|
||||
z.logger.Info("unblock URL", "url", url)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (z *ZscalerGateway) BlockCategory(_ context.Context, category string) error {
|
||||
z.logger.Info("block category", "category", category)
|
||||
// Stub: would update URL category policy to BLOCK
|
||||
return nil
|
||||
}
|
||||
|
||||
func (z *ZscalerGateway) HealthCheck(ctx context.Context) error {
|
||||
if z.cloudName == "" {
|
||||
return fmt.Errorf("not configured")
|
||||
}
|
||||
// Stub: would call GET /status
|
||||
return nil
|
||||
}
|
||||
|
||||
func (z *ZscalerGateway) Vendor() string { return "zscaler" }
|
||||
|
||||
// RegisterDefaultPlugins registers all built-in vendor plugin factories.
|
||||
func RegisterDefaultPlugins(registry *PluginRegistry) {
|
||||
registry.RegisterFactory(PluginTypeFirewall, "checkpoint", func() interface{} {
|
||||
return NewCheckPointEnforcer()
|
||||
})
|
||||
registry.RegisterFactory(PluginTypeEDR, "crowdstrike", func() interface{} {
|
||||
return NewCrowdStrikeController()
|
||||
})
|
||||
registry.RegisterFactory(PluginTypeProxy, "zscaler", func() interface{} {
|
||||
return NewZscalerGateway()
|
||||
})
|
||||
}
|
||||
212
internal/application/shadow_ai/registry.go
Normal file
212
internal/application/shadow_ai/registry.go
Normal file
|
|
@ -0,0 +1,212 @@
|
|||
package shadow_ai
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// PluginFactory creates a new plugin instance.
|
||||
type PluginFactory func() interface{}
|
||||
|
||||
// PluginRegistry manages vendor plugin registration, loading, and lifecycle.
|
||||
// Thread-safe via sync.RWMutex.
|
||||
type PluginRegistry struct {
|
||||
mu sync.RWMutex
|
||||
plugins map[string]interface{} // vendor → plugin instance
|
||||
factories map[string]PluginFactory // "type_vendor" → factory
|
||||
configs map[string]*PluginConfig // vendor → config
|
||||
health map[string]*PluginHealth // vendor → health status
|
||||
logger *slog.Logger
|
||||
}
|
||||
|
||||
// NewPluginRegistry creates a new plugin registry.
|
||||
func NewPluginRegistry() *PluginRegistry {
|
||||
return &PluginRegistry{
|
||||
plugins: make(map[string]interface{}),
|
||||
factories: make(map[string]PluginFactory),
|
||||
configs: make(map[string]*PluginConfig),
|
||||
health: make(map[string]*PluginHealth),
|
||||
logger: slog.Default().With("component", "shadow-ai-registry"),
|
||||
}
|
||||
}
|
||||
|
||||
// RegisterFactory registers a plugin factory for a given type+vendor combination.
|
||||
// Example: RegisterFactory("firewall", "checkpoint", func() interface{} { return &CheckPointEnforcer{} })
|
||||
func (r *PluginRegistry) RegisterFactory(pluginType PluginType, vendor string, factory PluginFactory) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
|
||||
key := fmt.Sprintf("%s_%s", pluginType, vendor)
|
||||
r.factories[key] = factory
|
||||
r.logger.Info("factory registered", "type", pluginType, "vendor", vendor)
|
||||
}
|
||||
|
||||
// LoadPlugins creates and initializes plugins from configuration.
|
||||
// Plugins that fail to initialize are logged but do not block other plugins.
|
||||
func (r *PluginRegistry) LoadPlugins(config *IntegrationConfig) error {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
|
||||
loaded := 0
|
||||
for i := range config.Plugins {
|
||||
pluginCfg := &config.Plugins[i]
|
||||
if !pluginCfg.Enabled {
|
||||
r.logger.Debug("plugin disabled, skipping", "vendor", pluginCfg.Vendor)
|
||||
continue
|
||||
}
|
||||
|
||||
key := fmt.Sprintf("%s_%s", pluginCfg.Type, pluginCfg.Vendor)
|
||||
factory, exists := r.factories[key]
|
||||
if !exists {
|
||||
r.logger.Warn("no factory for plugin", "key", key, "vendor", pluginCfg.Vendor)
|
||||
continue
|
||||
}
|
||||
|
||||
plugin := factory()
|
||||
|
||||
// Initialize if plugin supports it.
|
||||
if init, ok := plugin.(Initializer); ok {
|
||||
if err := init.Initialize(pluginCfg.Config); err != nil {
|
||||
r.logger.Error("plugin init failed", "vendor", pluginCfg.Vendor, "error", err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
r.plugins[pluginCfg.Vendor] = plugin
|
||||
r.configs[pluginCfg.Vendor] = pluginCfg
|
||||
r.health[pluginCfg.Vendor] = &PluginHealth{
|
||||
Vendor: pluginCfg.Vendor,
|
||||
Type: pluginCfg.Type,
|
||||
Status: PluginStatusHealthy,
|
||||
}
|
||||
loaded++
|
||||
r.logger.Info("plugin loaded", "vendor", pluginCfg.Vendor, "type", pluginCfg.Type)
|
||||
}
|
||||
|
||||
r.logger.Info("plugin loading complete", "loaded", loaded, "total", len(config.Plugins))
|
||||
return nil
|
||||
}
|
||||
|
||||
// Get returns a plugin by vendor name.
|
||||
func (r *PluginRegistry) Get(vendor string) (interface{}, bool) {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
p, ok := r.plugins[vendor]
|
||||
return p, ok
|
||||
}
|
||||
|
||||
// GetByType returns all plugins of a given type.
|
||||
func (r *PluginRegistry) GetByType(pluginType PluginType) []interface{} {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
|
||||
var result []interface{}
|
||||
for vendor, cfg := range r.configs {
|
||||
if cfg.Type == pluginType {
|
||||
if plugin, ok := r.plugins[vendor]; ok {
|
||||
result = append(result, plugin)
|
||||
}
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// GetNetworkEnforcers returns all loaded NetworkEnforcer plugins.
|
||||
func (r *PluginRegistry) GetNetworkEnforcers() []NetworkEnforcer {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
|
||||
var result []NetworkEnforcer
|
||||
for _, plugin := range r.plugins {
|
||||
if ne, ok := plugin.(NetworkEnforcer); ok {
|
||||
result = append(result, ne)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// GetEndpointControllers returns all loaded EndpointController plugins.
|
||||
func (r *PluginRegistry) GetEndpointControllers() []EndpointController {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
|
||||
var result []EndpointController
|
||||
for _, plugin := range r.plugins {
|
||||
if ec, ok := plugin.(EndpointController); ok {
|
||||
result = append(result, ec)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// GetWebGateways returns all loaded WebGateway plugins.
|
||||
func (r *PluginRegistry) GetWebGateways() []WebGateway {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
|
||||
var result []WebGateway
|
||||
for _, plugin := range r.plugins {
|
||||
if wg, ok := plugin.(WebGateway); ok {
|
||||
result = append(result, wg)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// IsHealthy returns true if a plugin is currently healthy.
|
||||
func (r *PluginRegistry) IsHealthy(vendor string) bool {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
h, ok := r.health[vendor]
|
||||
return ok && h.Status == PluginStatusHealthy
|
||||
}
|
||||
|
||||
// SetHealth updates the health status for a plugin.
|
||||
func (r *PluginRegistry) SetHealth(vendor string, health *PluginHealth) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.health[vendor] = health
|
||||
}
|
||||
|
||||
// GetHealth returns the health status snapshot for a plugin.
|
||||
func (r *PluginRegistry) GetHealth(vendor string) (*PluginHealth, bool) {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
h, ok := r.health[vendor]
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
cp := *h
|
||||
return &cp, true
|
||||
}
|
||||
|
||||
// AllHealth returns health snapshots for all plugins.
|
||||
func (r *PluginRegistry) AllHealth() []PluginHealth {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
|
||||
result := make([]PluginHealth, 0, len(r.health))
|
||||
for _, h := range r.health {
|
||||
result = append(result, *h)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// PluginCount returns the number of loaded plugins.
|
||||
func (r *PluginRegistry) PluginCount() int {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
return len(r.plugins)
|
||||
}
|
||||
|
||||
// Vendors returns all loaded vendor names.
|
||||
func (r *PluginRegistry) Vendors() []string {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
result := make([]string, 0, len(r.plugins))
|
||||
for v := range r.plugins {
|
||||
result = append(result, v)
|
||||
}
|
||||
return result
|
||||
}
|
||||
1225
internal/application/shadow_ai/shadow_ai_test.go
Normal file
1225
internal/application/shadow_ai/shadow_ai_test.go
Normal file
File diff suppressed because it is too large
Load diff
373
internal/application/shadow_ai/soc_integration.go
Normal file
373
internal/application/shadow_ai/soc_integration.go
Normal file
|
|
@ -0,0 +1,373 @@
|
|||
package shadow_ai
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ShadowAIController is the main orchestrator that ties together
|
||||
// detection, enforcement, SOC event emission, and statistics.
|
||||
type ShadowAIController struct {
|
||||
mu sync.RWMutex
|
||||
registry *PluginRegistry
|
||||
fallback *FallbackManager
|
||||
healthChecker *HealthChecker
|
||||
netDetector *NetworkDetector
|
||||
behavioral *BehavioralDetector
|
||||
docBridge *DocBridge
|
||||
approval *ApprovalEngine
|
||||
events []ShadowAIEvent // In-memory event store (bounded)
|
||||
maxEvents int
|
||||
socEventFn func(source, severity, category, description string, meta map[string]string) // Bridge to SOC event bus
|
||||
logger *slog.Logger
|
||||
}
|
||||
|
||||
// NewShadowAIController creates the main Shadow AI Control orchestrator.
|
||||
func NewShadowAIController() *ShadowAIController {
|
||||
registry := NewPluginRegistry()
|
||||
RegisterDefaultPlugins(registry)
|
||||
return &ShadowAIController{
|
||||
registry: registry,
|
||||
fallback: NewFallbackManager(registry, "detect_only"),
|
||||
netDetector: NewNetworkDetector(),
|
||||
behavioral: NewBehavioralDetector(100),
|
||||
docBridge: NewDocBridge(),
|
||||
approval: NewApprovalEngine(),
|
||||
events: make([]ShadowAIEvent, 0, 1000),
|
||||
maxEvents: 10000,
|
||||
logger: slog.Default().With("component", "shadow-ai-controller"),
|
||||
}
|
||||
}
|
||||
|
||||
// SetSOCEventEmitter sets the function used to emit events into the SOC pipeline.
|
||||
func (c *ShadowAIController) SetSOCEventEmitter(fn func(source, severity, category, description string, meta map[string]string)) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.socEventFn = fn
|
||||
}
|
||||
|
||||
// Configure loads plugin configuration and initializes the integration layer.
|
||||
func (c *ShadowAIController) Configure(config *IntegrationConfig) error {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
if err := c.registry.LoadPlugins(config); err != nil {
|
||||
return fmt.Errorf("failed to load plugins: %w", err)
|
||||
}
|
||||
|
||||
c.fallback = NewFallbackManager(c.registry, config.FallbackStrategy)
|
||||
c.fallback.SetEventLogger(func(event ShadowAIEvent) {
|
||||
c.recordEvent(event)
|
||||
})
|
||||
|
||||
interval := config.HealthCheckInterval
|
||||
if interval <= 0 {
|
||||
interval = 30 * time.Second
|
||||
}
|
||||
c.healthChecker = NewHealthChecker(c.registry, interval, func(vendor string, status PluginStatus, msg string) {
|
||||
c.emitSOCEvent("HIGH", "integration_health", msg, map[string]string{
|
||||
"vendor": vendor,
|
||||
"status": string(status),
|
||||
})
|
||||
})
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// StartHealthChecker starts continuous plugin health monitoring.
|
||||
func (c *ShadowAIController) StartHealthChecker(ctx context.Context) {
|
||||
if c.healthChecker != nil {
|
||||
go c.healthChecker.Start(ctx)
|
||||
}
|
||||
}
|
||||
|
||||
// ProcessNetworkEvent analyzes a network event and enforces policy.
|
||||
func (c *ShadowAIController) ProcessNetworkEvent(ctx context.Context, event NetworkEvent) *ShadowAIEvent {
|
||||
detected := c.netDetector.Analyze(event)
|
||||
if detected == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Record behavioral data.
|
||||
c.behavioral.RecordAccess(event.User, event.Destination, event.DataSize)
|
||||
|
||||
// Attempt to block.
|
||||
enforcedBy, err := c.fallback.BlockDomain(ctx, event.Destination, fmt.Sprintf("Shadow AI: %s", detected.AIService))
|
||||
if err != nil {
|
||||
c.logger.Error("enforcement failed", "destination", event.Destination, "error", err)
|
||||
}
|
||||
|
||||
if enforcedBy != "" {
|
||||
detected.Action = "blocked"
|
||||
detected.EnforcedBy = enforcedBy
|
||||
} else {
|
||||
detected.Action = "detected"
|
||||
}
|
||||
|
||||
detected.ID = genEventID()
|
||||
c.recordEvent(*detected)
|
||||
|
||||
// Emit to SOC event bus.
|
||||
c.emitSOCEvent("HIGH", "shadow_ai_usage",
|
||||
fmt.Sprintf("Shadow AI access detected: %s → %s", event.User, detected.AIService),
|
||||
map[string]string{
|
||||
"user": event.User,
|
||||
"hostname": event.Hostname,
|
||||
"destination": event.Destination,
|
||||
"ai_service": detected.AIService,
|
||||
"action": detected.Action,
|
||||
"enforced_by": detected.EnforcedBy,
|
||||
},
|
||||
)
|
||||
|
||||
return detected
|
||||
}
|
||||
|
||||
// ScanContent scans text content for AI API keys.
|
||||
func (c *ShadowAIController) ScanContent(content string) string {
|
||||
return c.netDetector.SignatureDB().ScanForAPIKeys(content)
|
||||
}
|
||||
|
||||
// ManualBlock manually blocks a domain or IP.
|
||||
func (c *ShadowAIController) ManualBlock(ctx context.Context, req BlockRequest) error {
|
||||
switch req.TargetType {
|
||||
case "domain":
|
||||
_, err := c.fallback.BlockDomain(ctx, req.Target, req.Reason)
|
||||
return err
|
||||
case "ip":
|
||||
_, err := c.fallback.BlockIP(ctx, req.Target, req.Duration, req.Reason)
|
||||
return err
|
||||
case "host":
|
||||
_, err := c.fallback.IsolateHost(ctx, req.Target)
|
||||
return err
|
||||
default:
|
||||
return fmt.Errorf("unsupported target type: %s", req.TargetType)
|
||||
}
|
||||
}
|
||||
|
||||
// GetStats returns aggregate shadow AI statistics.
|
||||
func (c *ShadowAIController) GetStats(timeRange string) ShadowAIStats {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
|
||||
cutoff := parseCutoff(timeRange)
|
||||
stats := ShadowAIStats{
|
||||
TimeRange: timeRange,
|
||||
ByService: make(map[string]int),
|
||||
ByDepartment: make(map[string]int),
|
||||
}
|
||||
|
||||
violatorMap := make(map[string]int)
|
||||
|
||||
for _, e := range c.events {
|
||||
if e.Timestamp.Before(cutoff) {
|
||||
continue
|
||||
}
|
||||
stats.Total++
|
||||
switch e.Action {
|
||||
case "blocked":
|
||||
stats.Blocked++
|
||||
case "allowed", "approved":
|
||||
stats.Approved++
|
||||
case "pending":
|
||||
stats.Pending++
|
||||
}
|
||||
if e.AIService != "" {
|
||||
stats.ByService[e.AIService]++
|
||||
}
|
||||
if dept, ok := e.Metadata["department"]; ok {
|
||||
stats.ByDepartment[dept]++
|
||||
}
|
||||
if e.UserID != "" {
|
||||
violatorMap[e.UserID]++
|
||||
}
|
||||
}
|
||||
|
||||
// Build top violators list (sorted desc).
|
||||
for uid, count := range violatorMap {
|
||||
stats.TopViolators = append(stats.TopViolators, Violator{UserID: uid, Attempts: count})
|
||||
}
|
||||
// Sort by attempts descending, limit to 10.
|
||||
for i := 0; i < len(stats.TopViolators); i++ {
|
||||
for j := i + 1; j < len(stats.TopViolators); j++ {
|
||||
if stats.TopViolators[j].Attempts > stats.TopViolators[i].Attempts {
|
||||
stats.TopViolators[i], stats.TopViolators[j] = stats.TopViolators[j], stats.TopViolators[i]
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(stats.TopViolators) > 10 {
|
||||
stats.TopViolators = stats.TopViolators[:10]
|
||||
}
|
||||
|
||||
return stats
|
||||
}
|
||||
|
||||
// GetEvents returns recent shadow AI events (newest first).
|
||||
func (c *ShadowAIController) GetEvents(limit int) []ShadowAIEvent {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
|
||||
total := len(c.events)
|
||||
if total == 0 {
|
||||
return nil
|
||||
}
|
||||
start := total - limit
|
||||
if start < 0 {
|
||||
start = 0
|
||||
}
|
||||
|
||||
// Return newest first.
|
||||
result := make([]ShadowAIEvent, 0, limit)
|
||||
for i := total - 1; i >= start; i-- {
|
||||
result = append(result, c.events[i])
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// GetEvent returns a single event by ID.
|
||||
func (c *ShadowAIController) GetEvent(id string) (*ShadowAIEvent, bool) {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
|
||||
for i := len(c.events) - 1; i >= 0; i-- {
|
||||
if c.events[i].ID == id {
|
||||
cp := c.events[i]
|
||||
return &cp, true
|
||||
}
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
|
||||
// IntegrationHealth returns health status of all plugins.
|
||||
func (c *ShadowAIController) IntegrationHealth() []PluginHealth {
|
||||
return c.registry.AllHealth()
|
||||
}
|
||||
|
||||
// VendorHealth returns health for a specific vendor.
|
||||
func (c *ShadowAIController) VendorHealth(vendor string) (*PluginHealth, bool) {
|
||||
return c.registry.GetHealth(vendor)
|
||||
}
|
||||
|
||||
// Registry returns the plugin registry for direct access.
|
||||
func (c *ShadowAIController) Registry() *PluginRegistry {
|
||||
return c.registry
|
||||
}
|
||||
|
||||
// NetworkDetector returns the network detector for configuration.
|
||||
func (c *ShadowAIController) NetworkDetector() *NetworkDetector {
|
||||
return c.netDetector
|
||||
}
|
||||
|
||||
// BehavioralDetector returns the behavioral detector.
|
||||
func (c *ShadowAIController) BehavioralDetector() *BehavioralDetector {
|
||||
return c.behavioral
|
||||
}
|
||||
|
||||
// DocBridge returns the document review bridge.
|
||||
func (c *ShadowAIController) DocBridge() *DocBridge {
|
||||
return c.docBridge
|
||||
}
|
||||
|
||||
// ApprovalEngine returns the approval workflow engine.
|
||||
func (c *ShadowAIController) ApprovalEngine() *ApprovalEngine {
|
||||
return c.approval
|
||||
}
|
||||
|
||||
// ReviewDocument scans a document and creates an approval request if needed.
|
||||
func (c *ShadowAIController) ReviewDocument(docID, content, userID string) (*ScanResult, *ApprovalRequest) {
|
||||
result := c.docBridge.ScanDocument(docID, content, userID)
|
||||
|
||||
// Create approval request based on data classification.
|
||||
var req *ApprovalRequest
|
||||
if result.Status != DocReviewBlocked {
|
||||
req = c.approval.SubmitRequest(userID, docID, result.DataClass)
|
||||
}
|
||||
|
||||
// Emit SOC event for tracking.
|
||||
c.emitSOCEvent("MEDIUM", "shadow_ai_usage",
|
||||
fmt.Sprintf("Document review: %s by %s — %s (%s)",
|
||||
docID, userID, result.Status, result.DataClass),
|
||||
map[string]string{
|
||||
"user": userID,
|
||||
"doc_id": docID,
|
||||
"status": string(result.Status),
|
||||
"data_class": string(result.DataClass),
|
||||
"pii_count": fmt.Sprintf("%d", len(result.PIIFound)),
|
||||
},
|
||||
)
|
||||
|
||||
return result, req
|
||||
}
|
||||
|
||||
// GenerateComplianceReport generates a compliance report for the given period.
|
||||
func (c *ShadowAIController) GenerateComplianceReport(period string) ComplianceReport {
|
||||
stats := c.GetStats(period)
|
||||
docStats := c.docBridge.Stats()
|
||||
return ComplianceReport{
|
||||
GeneratedAt: time.Now(),
|
||||
Period: period,
|
||||
TotalInteractions: stats.Total,
|
||||
BlockedAttempts: stats.Blocked,
|
||||
ApprovedReviews: stats.Approved,
|
||||
PIIDetected: docStats["redacted"] + docStats["blocked"],
|
||||
SecretsDetected: docStats["blocked"],
|
||||
AuditComplete: true,
|
||||
Regulations: []string{"GDPR", "SOC2", "EU AI Act Article 15"},
|
||||
}
|
||||
}
|
||||
|
||||
// --- Internal helpers ---
|
||||
|
||||
func (c *ShadowAIController) recordEvent(event ShadowAIEvent) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
c.events = append(c.events, event)
|
||||
|
||||
// Evict oldest events if over capacity.
|
||||
if len(c.events) > c.maxEvents {
|
||||
excess := len(c.events) - c.maxEvents
|
||||
c.events = c.events[excess:]
|
||||
}
|
||||
}
|
||||
|
||||
func (c *ShadowAIController) emitSOCEvent(severity, category, description string, meta map[string]string) {
|
||||
c.mu.RLock()
|
||||
fn := c.socEventFn
|
||||
c.mu.RUnlock()
|
||||
|
||||
if fn != nil {
|
||||
fn("shadow-ai", severity, category, description, meta)
|
||||
}
|
||||
}
|
||||
|
||||
func parseCutoff(timeRange string) time.Time {
|
||||
switch timeRange {
|
||||
case "1h":
|
||||
return time.Now().Add(-1 * time.Hour)
|
||||
case "24h":
|
||||
return time.Now().Add(-24 * time.Hour)
|
||||
case "7d":
|
||||
return time.Now().Add(-7 * 24 * time.Hour)
|
||||
case "30d":
|
||||
return time.Now().Add(-30 * 24 * time.Hour)
|
||||
case "90d":
|
||||
return time.Now().Add(-90 * 24 * time.Hour)
|
||||
default:
|
||||
return time.Now().Add(-24 * time.Hour)
|
||||
}
|
||||
}
|
||||
|
||||
var eventCounter uint64
|
||||
var eventCounterMu sync.Mutex
|
||||
|
||||
func genEventID() string {
|
||||
eventCounterMu.Lock()
|
||||
eventCounter++
|
||||
id := eventCounter
|
||||
eventCounterMu.Unlock()
|
||||
return fmt.Sprintf("sai-%d-%d", time.Now().UnixMilli(), id)
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue