feat: add structured audit event system (#1027)

Add a complete audit event pipeline that emits structured, machine-
parseable events for every gateway request and IAM decision.

Schema and publisher:
- AuditEvent dataclass and notify-class queue (audit_events_queue)
- AuditPublisher utility: fire-and-forget emission with envelope
  (schema_version, event_id, event_type, timestamp, producer)
- New request_id and client_ip fields on IamRequest for correlation

Gateway (gateway.request events):
- aiohttp middleware assigns request_id, captures timing/status/sizes
  and emits an event after every HTTP request completes
- IamAuth.authenticate annotates the request with identity
- Main endpoint handlers annotate capability and workspace
- request_id and client_ip forwarded to IAM on authenticate/authorise

IAM service (iam.authenticate, iam.authorise, iam.management events):
- Emits iam.authenticate for resolve-api-key, login, anonymous auth
- Emits iam.authorise for authorise and authorise-many decisions
- Emits iam.management for user/workspace/key mutations
- All events include request_id for correlation with gateway events

Design: events land on a pub/sub notify topic — non-persistent,
per-subscriber delivery. If no audit consumer is deployed, events
are silently discarded. Storage, retention, and alerting are
consumer-side concerns outside this boundary.

Added unit tests for the publisher and gateway middleware, unit
tests for IAM audit emission, and a contract test for the AuditEvent
schema.

Tech spec: docs/tech-specs/audit-events.md

tests/contract/test_schema_field_contracts.py

@@ -24,6 +24,8 @@ from trustgraph.schema import (
     EntityContext,
     EntityEmbeddings,
     ChunkEmbeddings,
+    AuditEvent,
+    IamRequest,
 )
 
 
@@ -72,3 +74,21 @@ class TestSchemaFieldContracts:
             "context",
             "chunk_id",
         }
+
+    def test_audit_event_fields(self):
+        assert _field_names(AuditEvent) == {
+            "schema_version",
+            "event_id",
+            "event_type",
+            "timestamp",
+            "producer",
+            "payload_json",
+        }
+
+    def test_iam_request_has_audit_fields(self):
+        """IamRequest must carry request_id and client_ip for audit
+        correlation.  Removing these breaks the gateway→IAM audit
+        chain."""
+        names = _field_names(IamRequest)
+        assert "request_id" in names
+        assert "client_ip" in names

tests/unit/test_base/test_audit_publisher.py

@@ -0,0 +1,90 @@
+"""
+Tests for the AuditPublisher utility.
+
+Verifies envelope construction, fire-and-forget semantics, and
+failure suppression.
+"""
+
+import json
+import pytest
+from unittest.mock import AsyncMock, MagicMock, patch
+
+from trustgraph.base.audit_publisher import AuditPublisher
+from trustgraph.schema import AuditEvent, audit_events_queue
+
+
+class TestAuditPublisherInit:
+
+    def test_queue_is_notify_class(self):
+        assert audit_events_queue == "notify:tg:audit-events"
+
+    def test_creates_producer_with_audit_queue(self):
+        backend = MagicMock()
+        pub = AuditPublisher(
+            backend=backend,
+            component_name="test-component",
+        )
+        assert pub.producer.topic == audit_events_queue
+        assert pub.producer.schema == AuditEvent
+        assert pub.component_name == "test-component"
+
+
+class TestAuditPublisherEmit:
+
+    @pytest.fixture
+    def publisher(self):
+        backend = MagicMock()
+        pub = AuditPublisher(
+            backend=backend,
+            component_name="test-svc",
+            processor_id="proc-1",
+        )
+        pub.producer = AsyncMock()
+        return pub
+
+    @pytest.mark.asyncio
+    async def test_emit_sends_structured_envelope(self, publisher):
+        await publisher.emit("gateway.request", {"path": "/test"})
+
+        publisher.producer.send.assert_called_once()
+        event = publisher.producer.send.call_args[0][0]
+
+        assert isinstance(event, AuditEvent)
+        assert event.schema_version == 1
+        assert event.event_type == "gateway.request"
+        assert event.producer == "test-svc"
+        assert event.event_id != ""
+        assert event.timestamp != ""
+
+    @pytest.mark.asyncio
+    async def test_emit_serializes_payload_as_json(self, publisher):
+        payload = {"method": "POST", "status_code": 200}
+        await publisher.emit("gateway.request", payload)
+
+        event = publisher.producer.send.call_args[0][0]
+        decoded = json.loads(event.payload_json)
+        assert decoded == payload
+
+    @pytest.mark.asyncio
+    async def test_emit_generates_unique_event_ids(self, publisher):
+        await publisher.emit("test.a", {})
+        await publisher.emit("test.b", {})
+
+        ids = [
+            call[0][0].event_id
+            for call in publisher.producer.send.call_args_list
+        ]
+        assert ids[0] != ids[1]
+
+    @pytest.mark.asyncio
+    async def test_emit_swallows_send_failure(self, publisher):
+        publisher.producer.send.side_effect = RuntimeError("pub/sub down")
+        await publisher.emit("test.event", {"key": "value"})
+
+    @pytest.mark.asyncio
+    async def test_emit_timestamp_is_utc_iso(self, publisher):
+        await publisher.emit("test.event", {})
+
+        event = publisher.producer.send.call_args[0][0]
+        assert "T" in event.timestamp
+        assert "+" in event.timestamp or "Z" in event.timestamp

tests/unit/test_gateway/test_audit_middleware.py

@@ -0,0 +1,198 @@
+"""
+Tests for the gateway audit middleware.
+
+Verifies that gateway.request events are emitted with correct
+metadata for success, error, and auth failure paths.
+"""
+
+import pytest
+from unittest.mock import AsyncMock, MagicMock
+from aiohttp import web
+from aiohttp.test_utils import make_mocked_request
+
+from trustgraph.gateway.audit import make_audit_middleware, _client_ip, _outcome_from_status
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+def _make_request(method="POST", path="/api/v1/config", headers=None,
+                  remote="10.0.0.1"):
+    hdrs = headers or {}
+    req = make_mocked_request(method, path, headers=hdrs)
+    req._transport_peername = (remote, 0)
+    return req
+
+
+# ---------------------------------------------------------------------------
+# Pure helpers
+# ---------------------------------------------------------------------------
+
+class TestClientIp:
+
+    def test_uses_x_forwarded_for(self):
+        req = _make_request(headers={"X-Forwarded-For": "1.2.3.4, 10.0.0.1"})
+        assert _client_ip(req) == "1.2.3.4"
+
+    def test_falls_back_to_remote(self):
+        req = _make_request(remote="192.168.1.1")
+        assert _client_ip(req) == "192.168.1.1"
+
+
+class TestOutcomeFromStatus:
+
+    def test_success_range(self):
+        assert _outcome_from_status(200) == "success"
+        assert _outcome_from_status(201) == "success"
+        assert _outcome_from_status(204) == "success"
+        assert _outcome_from_status(301) == "success"
+
+    def test_unauthenticated(self):
+        assert _outcome_from_status(401) == "unauthenticated"
+
+    def test_denied(self):
+        assert _outcome_from_status(403) == "denied"
+
+    def test_error(self):
+        assert _outcome_from_status(400) == "error"
+        assert _outcome_from_status(404) == "error"
+        assert _outcome_from_status(500) == "error"
+
+
+# ---------------------------------------------------------------------------
+# Middleware integration
+# ---------------------------------------------------------------------------
+
+class TestAuditMiddleware:
+
+    @pytest.fixture
+    def audit_publisher(self):
+        pub = AsyncMock()
+        pub.emit = AsyncMock()
+        return pub
+
+    @pytest.fixture
+    def middleware(self, audit_publisher):
+        return make_audit_middleware(audit_publisher)
+
+    @pytest.mark.asyncio
+    async def test_emits_event_on_success(self, middleware, audit_publisher):
+        async def handler(request):
+            return web.json_response({"ok": True})
+
+        req = _make_request()
+        await middleware(req, handler)
+
+        audit_publisher.emit.assert_called_once()
+        event_type, payload = audit_publisher.emit.call_args[0]
+        assert event_type == "gateway.request"
+        assert payload["method"] == "POST"
+        assert payload["path"] == "/api/v1/config"
+        assert payload["status_code"] == 200
+        assert payload["outcome"] == "success"
+        assert "request_id" in payload
+        assert "duration_ms" in payload
+        assert "error" not in payload
+
+    @pytest.mark.asyncio
+    async def test_emits_event_on_http_exception(self, middleware,
+                                                  audit_publisher):
+        async def handler(request):
+            raise web.HTTPForbidden(reason="access denied")
+
+        req = _make_request()
+        with pytest.raises(web.HTTPForbidden):
+            await middleware(req, handler)
+
+        audit_publisher.emit.assert_called_once()
+        _, payload = audit_publisher.emit.call_args[0]
+        assert payload["status_code"] == 403
+        assert payload["outcome"] == "denied"
+        assert payload["error"] == "access denied"
+
+    @pytest.mark.asyncio
+    async def test_emits_event_on_unhandled_exception(self, middleware,
+                                                       audit_publisher):
+        async def handler(request):
+            raise ValueError("boom")
+
+        req = _make_request()
+        with pytest.raises(ValueError):
+            await middleware(req, handler)
+
+        audit_publisher.emit.assert_called_once()
+        _, payload = audit_publisher.emit.call_args[0]
+        assert payload["status_code"] == 500
+        assert payload["outcome"] == "error"
+        assert payload["error"] == "ValueError"
+
+    @pytest.mark.asyncio
+    async def test_includes_identity_when_annotated(self, middleware,
+                                                     audit_publisher):
+        async def handler(request):
+            request['audit_identity'] = "user:mark"
+            return web.json_response({"ok": True})
+
+        req = _make_request()
+        await middleware(req, handler)
+
+        _, payload = audit_publisher.emit.call_args[0]
+        assert payload["identity"] == "user:mark"
+
+    @pytest.mark.asyncio
+    async def test_includes_capability_when_annotated(self, middleware,
+                                                       audit_publisher):
+        async def handler(request):
+            request['audit_capability'] = "config:read"
+            request['audit_workspace'] = "production"
+            return web.json_response({"ok": True})
+
+        req = _make_request()
+        await middleware(req, handler)
+
+        _, payload = audit_publisher.emit.call_args[0]
+        assert payload["capability"] == "config:read"
+        assert payload["workspace"] == "production"
+
+    @pytest.mark.asyncio
+    async def test_omits_unannotated_fields(self, middleware,
+                                             audit_publisher):
+        async def handler(request):
+            return web.json_response({"ok": True})
+
+        req = _make_request()
+        await middleware(req, handler)
+
+        _, payload = audit_publisher.emit.call_args[0]
+        assert "identity" not in payload
+        assert "capability" not in payload
+        assert "workspace" not in payload
+
+    @pytest.mark.asyncio
+    async def test_assigns_request_id(self, middleware, audit_publisher):
+        captured_id = None
+
+        async def handler(request):
+            nonlocal captured_id
+            captured_id = request.get('audit_request_id')
+            return web.json_response({"ok": True})
+
+        req = _make_request()
+        await middleware(req, handler)
+
+        assert captured_id is not None
+        _, payload = audit_publisher.emit.call_args[0]
+        assert payload["request_id"] == captured_id
+
+    @pytest.mark.asyncio
+    async def test_still_returns_response_if_emit_fails(self, middleware,
+                                                         audit_publisher):
+        audit_publisher.emit.side_effect = RuntimeError("pub/sub down")
+
+        async def handler(request):
+            return web.json_response({"ok": True})
+
+        req = _make_request()
+        resp = await middleware(req, handler)
+        assert resp.status == 200

tests/unit/test_gateway/test_auth.py

@@ -244,7 +244,7 @@ class TestIamAuthDispatch:
     async def test_api_key_path(self):
         auth = IamAuth(backend=Mock())
 
-        async def fake_resolve(api_key):
+        async def fake_resolve(api_key, **kwargs):
             assert api_key == "tg_testkey"
             # Roles are returned by the regime as a hint but the
             # gateway ignores them — kept here so the resolve
@@ -309,7 +309,7 @@ class TestApiKeyCache:
         seen = []
 
         async def fake_with_client(op):
-            async def resolve(plaintext):
+            async def resolve(plaintext, **kwargs):
                 seen.append(plaintext)
                 return ("u-" + plaintext, "default", ["reader"])
             return await op(Mock(resolve_api_key=resolve))

tests/unit/test_iam/test_iam_audit_events.py

@@ -0,0 +1,345 @@
+"""
+Tests for IAM service audit event emission.
+
+Verifies that the IAM Processor emits correctly categorised audit
+events (iam.authenticate, iam.authorise, iam.management) after
+handling each request type.
+"""
+
+import asyncio
+import pytest
+from unittest.mock import AsyncMock, MagicMock, Mock
+
+from trustgraph.schema import IamRequest, IamResponse, Error
+from trustgraph.iam.service.service import Processor
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+def _make_msg(request, msg_id="test-msg-1"):
+    msg = MagicMock()
+    msg.value.return_value = request
+    msg.properties.return_value = {"id": msg_id}
+    return msg
+
+
+def _make_processor():
+    """Create a Processor with stubbed dependencies."""
+    proc = object.__new__(Processor)
+    proc.id = "test-iam-svc"
+    proc.iam_response_producer = AsyncMock()
+    proc.audit = AsyncMock()
+    return proc
+
+
+# ---------------------------------------------------------------------------
+# Authentication events
+# ---------------------------------------------------------------------------
+
+class TestAuthenticateAuditEvents:
+
+    @pytest.mark.asyncio
+    async def test_resolve_api_key_success(self):
+        proc = _make_processor()
+        resp = IamResponse(
+            resolved_user_id="user-123",
+            resolved_default_workspace="default",
+        )
+        proc.iam = MagicMock()
+        proc.iam.handle = AsyncMock(return_value=resp)
+
+        req = IamRequest(
+            operation="resolve-api-key",
+            api_key="tg_test",
+            request_id="req-1",
+            client_ip="10.0.0.1",
+        )
+        await proc.on_iam_request(_make_msg(req), None, None)
+
+        proc.audit.emit.assert_called_once()
+        event_type, payload = proc.audit.emit.call_args[0]
+        assert event_type == "iam.authenticate"
+        assert payload["credential_type"] == "api-key"
+        assert payload["identity"] == "user-123"
+        assert payload["outcome"] == "success"
+        assert payload["request_id"] == "req-1"
+        assert payload["client_ip"] == "10.0.0.1"
+        assert "failure_reason" not in payload
+
+    @pytest.mark.asyncio
+    async def test_resolve_api_key_failure(self):
+        proc = _make_processor()
+        resp = IamResponse(error=Error(type="auth-failed", message="unknown"))
+        proc.iam = MagicMock()
+        proc.iam.handle = AsyncMock(return_value=resp)
+
+        req = IamRequest(
+            operation="resolve-api-key",
+            api_key="tg_bad",
+            request_id="req-2",
+            client_ip="10.0.0.2",
+        )
+        await proc.on_iam_request(_make_msg(req), None, None)
+
+        _, payload = proc.audit.emit.call_args[0]
+        assert payload["outcome"] == "failure"
+        assert payload["identity"] == "unknown"
+        assert payload["failure_reason"] == "auth-failed"
+
+    @pytest.mark.asyncio
+    async def test_login_emits_authenticate(self):
+        proc = _make_processor()
+        resp = IamResponse(
+            resolved_user_id="user-456",
+            jwt="eyJ...",
+            jwt_expires="2026-07-06T10:00:00Z",
+        )
+        proc.iam = MagicMock()
+        proc.iam.handle = AsyncMock(return_value=resp)
+
+        req = IamRequest(
+            operation="login",
+            username="mark",
+            password="secret",
+            request_id="req-3",
+        )
+        await proc.on_iam_request(_make_msg(req), None, None)
+
+        event_type, payload = proc.audit.emit.call_args[0]
+        assert event_type == "iam.authenticate"
+        assert payload["credential_type"] == "login-password"
+
+    @pytest.mark.asyncio
+    async def test_anonymous_emits_authenticate(self):
+        proc = _make_processor()
+        resp = IamResponse(error=Error(type="auth-failed", message="no"))
+        proc.iam = MagicMock()
+        proc.iam.handle = AsyncMock(return_value=resp)
+
+        req = IamRequest(operation="authenticate-anonymous")
+        await proc.on_iam_request(_make_msg(req), None, None)
+
+        event_type, payload = proc.audit.emit.call_args[0]
+        assert event_type == "iam.authenticate"
+        assert payload["credential_type"] == "anonymous"
+        assert payload["outcome"] == "failure"
+
+
+# ---------------------------------------------------------------------------
+# Authorise events
+# ---------------------------------------------------------------------------
+
+class TestAuthoriseAuditEvents:
+
+    @pytest.mark.asyncio
+    async def test_authorise_allow(self):
+        proc = _make_processor()
+        resp = IamResponse(decision_allow=True, decision_ttl_seconds=60)
+        proc.iam = MagicMock()
+        proc.iam.handle = AsyncMock(return_value=resp)
+
+        req = IamRequest(
+            operation="authorise",
+            user_id="user-1",
+            capability="config:read",
+            resource_json='{"workspace": "production"}',
+            request_id="req-4",
+        )
+        await proc.on_iam_request(_make_msg(req), None, None)
+
+        event_type, payload = proc.audit.emit.call_args[0]
+        assert event_type == "iam.authorise"
+        assert payload["outcome"] == "allow"
+        assert payload["capability"] == "config:read"
+        assert payload["identity"] == "user-1"
+        assert payload["workspace"] == "production"
+        assert "denial_reason" not in payload
+
+    @pytest.mark.asyncio
+    async def test_authorise_deny(self):
+        proc = _make_processor()
+        resp = IamResponse(decision_allow=False, decision_ttl_seconds=5)
+        proc.iam = MagicMock()
+        proc.iam.handle = AsyncMock(return_value=resp)
+
+        req = IamRequest(
+            operation="authorise",
+            user_id="user-2",
+            capability="users:admin",
+            resource_json='{}',
+            request_id="req-5",
+        )
+        await proc.on_iam_request(_make_msg(req), None, None)
+
+        _, payload = proc.audit.emit.call_args[0]
+        assert payload["outcome"] == "deny"
+        assert payload["denial_reason"] == "capability-not-in-role"
+
+    @pytest.mark.asyncio
+    async def test_authorise_extracts_workspace_from_resource_json(self):
+        proc = _make_processor()
+        resp = IamResponse(decision_allow=True, decision_ttl_seconds=60)
+        proc.iam = MagicMock()
+        proc.iam.handle = AsyncMock(return_value=resp)
+
+        req = IamRequest(
+            operation="authorise",
+            user_id="user-1",
+            capability="graph-rag:query",
+            resource_json='{"workspace": "engineering"}',
+        )
+        await proc.on_iam_request(_make_msg(req), None, None)
+
+        _, payload = proc.audit.emit.call_args[0]
+        assert payload["workspace"] == "engineering"
+
+    @pytest.mark.asyncio
+    async def test_authorise_omits_empty_workspace(self):
+        proc = _make_processor()
+        resp = IamResponse(decision_allow=True, decision_ttl_seconds=60)
+        proc.iam = MagicMock()
+        proc.iam.handle = AsyncMock(return_value=resp)
+
+        req = IamRequest(
+            operation="authorise",
+            user_id="user-1",
+            capability="metrics:read",
+            resource_json='{}',
+        )
+        await proc.on_iam_request(_make_msg(req), None, None)
+
+        _, payload = proc.audit.emit.call_args[0]
+        assert "workspace" not in payload
+
+
+# ---------------------------------------------------------------------------
+# Management events
+# ---------------------------------------------------------------------------
+
+class TestManagementAuditEvents:
+
+    @pytest.mark.asyncio
+    async def test_create_user_emits_management(self):
+        proc = _make_processor()
+        resp = IamResponse()
+        proc.iam = MagicMock()
+        proc.iam.handle = AsyncMock(return_value=resp)
+
+        req = IamRequest(
+            operation="create-user",
+            actor="admin-1",
+            user_id="new-user",
+            workspace="default",
+            request_id="req-6",
+        )
+        await proc.on_iam_request(_make_msg(req), None, None)
+
+        event_type, payload = proc.audit.emit.call_args[0]
+        assert event_type == "iam.management"
+        assert payload["operation"] == "create-user"
+        assert payload["actor"] == "admin-1"
+        assert payload["target_identity"] == "new-user"
+        assert payload["target_workspace"] == "default"
+        assert payload["outcome"] == "success"
+
+    @pytest.mark.asyncio
+    async def test_revoke_api_key_emits_management(self):
+        proc = _make_processor()
+        resp = IamResponse()
+        proc.iam = MagicMock()
+        proc.iam.handle = AsyncMock(return_value=resp)
+
+        req = IamRequest(
+            operation="revoke-api-key",
+            actor="admin-1",
+            key_id="key-abc",
+            workspace="production",
+            request_id="req-7",
+        )
+        await proc.on_iam_request(_make_msg(req), None, None)
+
+        event_type, payload = proc.audit.emit.call_args[0]
+        assert event_type == "iam.management"
+        assert payload["operation"] == "revoke-api-key"
+        assert payload["outcome"] == "success"
+
+    @pytest.mark.asyncio
+    async def test_management_error_outcome(self):
+        proc = _make_processor()
+        resp = IamResponse(
+            error=Error(type="not-found", message="user not found"),
+        )
+        proc.iam = MagicMock()
+        proc.iam.handle = AsyncMock(return_value=resp)
+
+        req = IamRequest(
+            operation="delete-user",
+            actor="admin-1",
+            user_id="ghost",
+        )
+        await proc.on_iam_request(_make_msg(req), None, None)
+
+        _, payload = proc.audit.emit.call_args[0]
+        assert payload["outcome"] == "error"
+
+    @pytest.mark.asyncio
+    async def test_management_omits_empty_target_fields(self):
+        proc = _make_processor()
+        resp = IamResponse()
+        proc.iam = MagicMock()
+        proc.iam.handle = AsyncMock(return_value=resp)
+
+        req = IamRequest(
+            operation="rotate-signing-key",
+            actor="admin-1",
+        )
+        await proc.on_iam_request(_make_msg(req), None, None)
+
+        _, payload = proc.audit.emit.call_args[0]
+        assert "target_identity" not in payload
+        assert "target_workspace" not in payload
+
+
+# ---------------------------------------------------------------------------
+# Non-audited operations
+# ---------------------------------------------------------------------------
+
+class TestNonAuditedOperations:
+
+    @pytest.mark.asyncio
+    async def test_whoami_does_not_emit(self):
+        proc = _make_processor()
+        resp = IamResponse()
+        proc.iam = MagicMock()
+        proc.iam.handle = AsyncMock(return_value=resp)
+
+        req = IamRequest(operation="whoami", actor="user-1")
+        await proc.on_iam_request(_make_msg(req), None, None)
+
+        proc.audit.emit.assert_not_called()
+
+    @pytest.mark.asyncio
+    async def test_list_users_does_not_emit(self):
+        proc = _make_processor()
+        resp = IamResponse()
+        proc.iam = MagicMock()
+        proc.iam.handle = AsyncMock(return_value=resp)
+
+        req = IamRequest(operation="list-users", workspace="default")
+        await proc.on_iam_request(_make_msg(req), None, None)
+
+        proc.audit.emit.assert_not_called()
+
+    @pytest.mark.asyncio
+    async def test_get_signing_key_does_not_emit(self):
+        proc = _make_processor()
+        resp = IamResponse(signing_key_public="PEM...")
+        proc.iam = MagicMock()
+        proc.iam.handle = AsyncMock(return_value=resp)
+
+        req = IamRequest(operation="get-signing-key-public")
+        await proc.on_iam_request(_make_msg(req), None, None)
+
+        proc.audit.emit.assert_not_called()