feat: add structured audit event system (#1027)

Add a complete audit event pipeline that emits structured, machine-
parseable events for every gateway request and IAM decision.

Schema and publisher:
- AuditEvent dataclass and notify-class queue (audit_events_queue)
- AuditPublisher utility: fire-and-forget emission with envelope
  (schema_version, event_id, event_type, timestamp, producer)
- New request_id and client_ip fields on IamRequest for correlation

Gateway (gateway.request events):
- aiohttp middleware assigns request_id, captures timing/status/sizes
  and emits an event after every HTTP request completes
- IamAuth.authenticate annotates the request with identity
- Main endpoint handlers annotate capability and workspace
- request_id and client_ip forwarded to IAM on authenticate/authorise

IAM service (iam.authenticate, iam.authorise, iam.management events):
- Emits iam.authenticate for resolve-api-key, login, anonymous auth
- Emits iam.authorise for authorise and authorise-many decisions
- Emits iam.management for user/workspace/key mutations
- All events include request_id for correlation with gateway events

Design: events land on a pub/sub notify topic — non-persistent,
per-subscriber delivery. If no audit consumer is deployed, events
are silently discarded. Storage, retention, and alerting are
consumer-side concerns outside this boundary.

Added unit tests for the publisher and gateway middleware, unit
tests for IAM audit emission, and a contract test for the AuditEvent
schema.

Tech spec: docs/tech-specs/audit-events.md

trustgraph-flow/trustgraph/gateway/audit.py

@@ -0,0 +1,105 @@
+"""
+Audit middleware for the API gateway.
+
+Wraps every HTTP request with timing and metadata collection, then
+emits a ``gateway.request`` audit event after the response is sent.
+Handlers can enrich the event by annotating the request dict:
+
+    request['audit_identity'] = identity.principal_id
+    request['audit_capability'] = capability
+    request['audit_workspace'] = workspace
+"""
+
+import time
+import logging
+from uuid import uuid4
+
+from aiohttp import web
+
+from trustgraph.base import AuditPublisher
+
+logger = logging.getLogger("audit")
+
+
+def _client_ip(request):
+    forwarded = request.headers.get("X-Forwarded-For")
+    if forwarded:
+        return forwarded.split(",")[0].strip()
+    peer = request.remote
+    return peer or ""
+
+
+def _outcome_from_status(status):
+    if 200 <= status < 400:
+        return "success"
+    if status == 401:
+        return "unauthenticated"
+    if status == 403:
+        return "denied"
+    return "error"
+
+
+def make_audit_middleware(audit_publisher):
+
+    @web.middleware
+    async def audit_middleware(request, handler):
+        request_id = str(uuid4())
+        request['audit_request_id'] = request_id
+        start = time.monotonic()
+
+        status_code = 500
+        error = None
+        response_size = 0
+
+        try:
+            response = await handler(request)
+            status_code = response.status
+            response_size = response.content_length or 0
+            return response
+        except web.HTTPException as exc:
+            status_code = exc.status
+            error = exc.reason
+            raise
+        except Exception as exc:
+            status_code = 500
+            error = type(exc).__name__
+            raise
+        finally:
+            duration_ms = int((time.monotonic() - start) * 1000)
+            outcome = _outcome_from_status(status_code)
+
+            payload = {
+                "request_id": request_id,
+                "method": request.method,
+                "path": request.path,
+                "client_ip": _client_ip(request),
+                "user_agent": request.headers.get("User-Agent", ""),
+                "status_code": status_code,
+                "outcome": outcome,
+                "duration_ms": duration_ms,
+                "request_size_bytes": request.content_length or 0,
+                "response_size_bytes": response_size,
+            }
+
+            identity = request.get('audit_identity')
+            if identity:
+                payload["identity"] = identity
+
+            capability = request.get('audit_capability')
+            if capability:
+                payload["capability"] = capability
+
+            workspace = request.get('audit_workspace')
+            if workspace:
+                payload["workspace"] = workspace
+
+            if error and outcome != "success":
+                payload["error"] = error
+
+            try:
+                await audit_publisher.emit("gateway.request", payload)
+            except Exception:
+                logger.debug("Failed to emit gateway audit event",
+                             exc_info=True)
+
+    return audit_middleware

trustgraph-flow/trustgraph/gateway/auth.py

@@ -232,21 +232,55 @@ class IamAuth:
         cannot distinguish missing / malformed / invalid / expired /
         revoked credentials."""
 
+        request_id = request.get('audit_request_id', '') if hasattr(request, 'get') else ''
+        client_ip = self._extract_client_ip(request)
+
         header = request.headers.get("Authorization", "")
         if not header.startswith("Bearer "):
-            return await self._authenticate_anonymous()
+            identity = await self._authenticate_anonymous(
+                request_id=request_id, client_ip=client_ip,
+            )
+            self._annotate_request(request, identity)
+            return identity
         token = header[len("Bearer "):].strip()
         if not token:
-            return await self._authenticate_anonymous()
+            identity = await self._authenticate_anonymous(
+                request_id=request_id, client_ip=client_ip,
+            )
+            self._annotate_request(request, identity)
+            return identity
 
         # API keys always start with "tg_".  JWTs have two dots and
         # no "tg_" prefix.  Discriminate cheaply.
         if token.startswith("tg_"):
-            return await self._resolve_api_key(token)
+            identity = await self._resolve_api_key(
+                token, request_id=request_id, client_ip=client_ip,
+            )
+            self._annotate_request(request, identity)
+            return identity
         if token.count(".") == 2:
-            return self._verify_jwt(token)
+            identity = self._verify_jwt(token)
+            self._annotate_request(request, identity)
+            return identity
         raise _auth_failure()
 
+    @staticmethod
+    def _extract_client_ip(request):
+        try:
+            forwarded = request.headers.get("X-Forwarded-For")
+            if forwarded:
+                return forwarded.split(",")[0].strip()
+            return request.remote or ""
+        except Exception:
+            return ""
+
+    @staticmethod
+    def _annotate_request(request, identity):
+        try:
+            request['audit_identity'] = identity.principal_id
+        except Exception:
+            pass
+
     def _verify_jwt(self, token):
         if not self._signing_public_pem:
             raise _auth_failure()
@@ -266,10 +300,12 @@ class IamAuth:
             principal_id=sub, source="jwt",
         )
 
-    async def _authenticate_anonymous(self):
+    async def _authenticate_anonymous(self, request_id="", client_ip=""):
         try:
             async def _call(client):
-                return await client.authenticate_anonymous()
+                return await client.authenticate_anonymous(
+                    request_id=request_id, client_ip=client_ip,
+                )
             user_id, default_workspace, _roles = await self._with_client(
                 _call,
             )
@@ -288,7 +324,7 @@ class IamAuth:
             principal_id=user_id, source="anonymous",
         )
 
-    async def _resolve_api_key(self, plaintext):
+    async def _resolve_api_key(self, plaintext, request_id="", client_ip=""):
         h = hashlib.sha256(plaintext.encode("utf-8")).hexdigest()
 
         cached = self._key_cache.get(h)
@@ -303,7 +339,10 @@ class IamAuth:
 
             try:
                 async def _call(client):
-                    return await client.resolve_api_key(plaintext)
+                    return await client.resolve_api_key(
+                        plaintext, request_id=request_id,
+                        client_ip=client_ip,
+                    )
                 # ``roles`` is returned by the OSS regime as a hint
                 # but is not consulted by the gateway; all policy
                 # decisions go through ``authorise``.
@@ -345,7 +384,8 @@ class IamAuth:
         )
         return hashlib.sha256(payload.encode("utf-8")).hexdigest()
 
-    async def authorise(self, identity, capability, resource, parameters):
+    async def authorise(self, identity, capability, resource, parameters,
+                        request_id="", client_ip=""):
         """Ask the IAM regime whether ``identity`` may perform
         ``capability`` on ``resource`` given ``parameters``.
 
@@ -383,6 +423,7 @@ class IamAuth:
                     return await client.authorise(
                         identity.handle, capability,
                         resource or {}, parameters or {},
+                        request_id=request_id, client_ip=client_ip,
                     )
                 allow, ttl = await self._with_client(_call)
             except Exception as e:

trustgraph-flow/trustgraph/gateway/endpoint/iam_endpoint.py

@@ -90,7 +90,13 @@ class IamEndpoint:
 
             await self.auth.authorise(
                 identity, op.capability, resource, parameters,
+                request_id=request.get('audit_request_id', ''),
+                client_ip=self.auth._extract_client_ip(request),
             )
+            request['audit_capability'] = op.capability
+            ws = resource.get('workspace', '')
+            if ws:
+                request['audit_workspace'] = ws
 
         # Plumb the authenticated caller's handle through as ``actor``
         # so iam-svc handlers (e.g. whoami, future actor-scoped

trustgraph-flow/trustgraph/gateway/endpoint/manager.py

@@ -75,9 +75,14 @@ class _RoutedVariableEndpoint:
             parameters = op.extract_parameters(ctx)
             await self.auth.authorise(
                 identity, op.capability, resource, parameters,
+                request_id=request.get('audit_request_id', ''),
+                client_ip=self.auth._extract_client_ip(request),
             )
+            request['audit_capability'] = op.capability
 
             ws = resource.get("workspace", "")
+            if ws:
+                request['audit_workspace'] = ws
             if ws and ws not in self.auth.known_workspaces:
                 raise workspace_not_found()
 
@@ -143,9 +148,14 @@ class _RoutedSocketEndpoint:
             parameters = op.extract_parameters(ctx)
             await self.auth.authorise(
                 identity, op.capability, resource, parameters,
+                request_id=request.get('audit_request_id', ''),
+                client_ip=self.auth._extract_client_ip(request),
             )
+            request['audit_capability'] = op.capability
 
             ws = resource.get("workspace", "")
+            if ws:
+                request['audit_workspace'] = ws
             if ws and ws not in self.auth.known_workspaces:
                 raise workspace_not_found()
 

trustgraph-flow/trustgraph/gateway/endpoint/registry_endpoint.py

@@ -98,7 +98,13 @@ class RegistryRoutedVariableEndpoint:
 
             await self.auth.authorise(
                 identity, op.capability, resource, parameters,
+                request_id=request.get('audit_request_id', ''),
+                client_ip=self.auth._extract_client_ip(request),
             )
+            request['audit_capability'] = op.capability
+            ws = resource.get('workspace', '')
+            if ws:
+                request['audit_workspace'] = ws
 
             # Default-fill workspace into the body so downstream
             # dispatchers see the canonical resolved value.  The

trustgraph-flow/trustgraph/gateway/service.py

@@ -11,8 +11,10 @@ import os
 
 from trustgraph.base.logging import setup_logging, add_logging_args
 from trustgraph.base.pubsub import get_pubsub, add_pubsub_args
+from trustgraph.base import AuditPublisher
 
 from . auth import IamAuth
+from . audit import make_audit_middleware
 from . config.receiver import ConfigReceiver
 from . dispatch.manager import DispatcherManager
 
@@ -122,12 +124,18 @@ class Api:
             timeout = self.timeout,
         )
 
+        self.audit_publisher = AuditPublisher(
+            backend=self.pubsub_backend,
+            component_name="api-gateway",
+            processor_id=config.get("id", "api-gateway"),
+        )
+
         self.endpoint_manager = EndpointManager(
             dispatcher_manager = self.dispatcher_manager,
             auth = self.auth,
             prometheus_url = self.prometheus_url,
             timeout = self.timeout,
-            
+
         )
 
         self.endpoints = [
@@ -136,7 +144,7 @@ class Api:
     async def app_factory(self):
 
         self.app = web.Application(
-            middlewares=[],
+            middlewares=[make_audit_middleware(self.audit_publisher)],
             client_max_size=256 * 1024 * 1024
         )
 

trustgraph-flow/trustgraph/iam/service/service.py

@@ -16,6 +16,7 @@ from trustgraph.schema import ConfigRequest, ConfigResponse, ConfigValue
 from trustgraph.schema import config_request_queue, config_response_queue
 
 from trustgraph.base import AsyncProcessor, Consumer, Producer
+from trustgraph.base import AuditPublisher
 from trustgraph.base import ConsumerMetrics, ProducerMetrics
 from trustgraph.base.metrics import SubscriberMetrics
 from trustgraph.base.request_response_spec import RequestResponse
@@ -145,6 +146,12 @@ class Processor(AsyncProcessor):
             metrics=iam_response_metrics,
         )
 
+        self.audit = AuditPublisher(
+            backend=self.pubsub,
+            component_name="iam-service",
+            processor_id=self.id,
+        )
+
         self.iam = IamService(
             host=self.cassandra_host,
             username=self.cassandra_username,
@@ -243,6 +250,19 @@ class Processor(AsyncProcessor):
                 f"{workspace_id}: {e}", exc_info=True,
             )
 
+    AUTHENTICATE_OPS = frozenset({
+        "resolve-api-key", "login", "authenticate-anonymous",
+    })
+    AUTHORISE_OPS = frozenset({
+        "authorise", "authorise-many",
+    })
+    MANAGEMENT_OPS = frozenset({
+        "create-user", "update-user", "disable-user", "enable-user",
+        "delete-user", "create-api-key", "revoke-api-key",
+        "create-workspace", "update-workspace", "disable-workspace",
+        "reset-password", "rotate-signing-key", "bootstrap",
+    })
+
     async def on_iam_request(self, msg, consumer, flow):
 
         id = None
@@ -256,6 +276,7 @@ class Processor(AsyncProcessor):
             await self.iam_response_producer.send(
                 resp, properties={"id": id},
             )
+            await self._emit_audit(v, resp)
         except Exception as e:
             logger.error(
                 f"IAM request failed: {type(e).__name__}: {e}",
@@ -269,6 +290,76 @@ class Processor(AsyncProcessor):
                     resp, properties={"id": id},
                 )
 
+    async def _emit_audit(self, v, resp):
+        try:
+            op = v.operation
+            if op in self.AUTHENTICATE_OPS:
+                await self._emit_authenticate(v, resp)
+            elif op in self.AUTHORISE_OPS:
+                await self._emit_authorise(v, resp)
+            elif op in self.MANAGEMENT_OPS:
+                await self._emit_management(v, resp)
+        except Exception:
+            logger.debug("Failed to emit IAM audit event", exc_info=True)
+
+    async def _emit_authenticate(self, v, resp):
+        has_error = resp.error is not None
+        payload = {
+            "request_id": v.request_id,
+            "credential_type": self._credential_type(v.operation),
+            "identity": resp.resolved_user_id if not has_error else "unknown",
+            "outcome": "failure" if has_error else "success",
+            "client_ip": v.client_ip,
+        }
+        if has_error:
+            payload["failure_reason"] = resp.error.type
+        if v.key_id:
+            payload["key_id"] = v.key_id
+        await self.audit.emit("iam.authenticate", payload)
+
+    async def _emit_authorise(self, v, resp):
+        import json as _json
+        workspace = v.workspace
+        if not workspace:
+            try:
+                resource = _json.loads(v.resource_json or "{}")
+                workspace = resource.get("workspace", "")
+            except Exception:
+                pass
+        payload = {
+            "request_id": v.request_id,
+            "identity": v.user_id,
+            "capability": v.capability,
+            "outcome": "allow" if resp.decision_allow else "deny",
+        }
+        if workspace:
+            payload["workspace"] = workspace
+        if not resp.decision_allow:
+            payload["denial_reason"] = "capability-not-in-role"
+        await self.audit.emit("iam.authorise", payload)
+
+    async def _emit_management(self, v, resp):
+        has_error = resp.error is not None
+        payload = {
+            "request_id": v.request_id,
+            "actor": v.actor,
+            "operation": v.operation,
+            "outcome": "error" if has_error else "success",
+        }
+        if v.user_id:
+            payload["target_identity"] = v.user_id
+        if v.workspace:
+            payload["target_workspace"] = v.workspace
+        await self.audit.emit("iam.management", payload)
+
+    @staticmethod
+    def _credential_type(operation):
+        if operation == "resolve-api-key":
+            return "api-key"
+        if operation == "login":
+            return "login-password"
+        return "anonymous"
+
     @staticmethod
     def add_args(parser):
         AsyncProcessor.add_args(parser)