feat: hybrid retrieval (BM25 + vector RRF fusion) for document-RAG (#875) (#1030)

Adds a sparse keyword retrieval path beside the existing vector path in
document-RAG, fused by weighted Reciprocal Rank Fusion on chunk_id, behind
--retrieval-mode (vector | keyword | hybrid, default vector).

The keyword index is a new pluggable service (KeywordIndexService /
KeywordIndexClientSpec); the first backend is SQLite FTS5, consuming Chunk
messages off the ingestion stream and answering BM25 queries from one
process, since the index is a single local file. Query text is sanitized
into per-term quoted phrases (raw text is not valid FTS5 syntax), which
also makes dotted clause numbers and error codes exact-match without a
trigram index. Indexes are scoped per (workspace, collection) and dropped
on collection deletion.

The keyword-index client spec is only registered when the sparse path is
enabled, so existing flow definitions without keyword-index queues are
untouched; with retrieval_mode=vector the retrieval path is unchanged. In
hybrid mode a keyword-path failure degrades to vector-only.

tests/unit/test_base/test_optional_request_response_spec.py

@@ -0,0 +1,81 @@
+"""
+Tests for RequestResponseSpec's optional flag: an optional client spec
+binds only when the flow definition declares its topics, so a definition
+predating the topics skips the binding (flow(name) then returns None)
+instead of raising KeyError during Flow construction — which would wedge
+the processor's start-flow retry loop.
+"""
+
+import pytest
+from unittest.mock import MagicMock
+
+from trustgraph.base.request_response_spec import RequestResponseSpec
+
+
+class StubImpl:
+    """Captures constructor kwargs; stands in for RequestResponse."""
+
+    def __init__(self, **kwargs):
+        self.kwargs = kwargs
+
+
+def make_spec(optional):
+    return RequestResponseSpec(
+        request_name="keyword-index-request",
+        request_schema=object,
+        response_name="keyword-index-response",
+        response_schema=object,
+        impl=StubImpl,
+        optional=optional,
+    )
+
+
+def make_flow():
+    flow = MagicMock()
+    flow.id = "f-id"
+    flow.name = "f-name"
+    flow.workspace = "ws"
+    flow.consumer = {}
+    return flow
+
+
+FULL_TOPICS = {
+    "topics": {
+        "keyword-index-request": "request:tg:keyword-index:ws:f",
+        "keyword-index-response": "response:tg:keyword-index:ws:f",
+    }
+}
+
+
+class TestOptionalRequestResponseSpec:
+
+    def test_optional_spec_skips_binding_when_topics_absent(self):
+        flow = make_flow()
+        make_spec(optional=True).add(flow, MagicMock(), {"topics": {}})
+        assert flow.consumer == {}
+
+    def test_optional_spec_skips_when_only_one_topic_present(self):
+        flow = make_flow()
+        definition = {
+            "topics": {
+                "keyword-index-request": "request:tg:keyword-index:ws:f",
+            }
+        }
+        make_spec(optional=True).add(flow, MagicMock(), definition)
+        assert flow.consumer == {}
+
+    def test_optional_spec_binds_when_topics_present(self):
+        flow = make_flow()
+        make_spec(optional=True).add(flow, MagicMock(), FULL_TOPICS)
+        client = flow.consumer["keyword-index-request"]
+        assert isinstance(client, StubImpl)
+        assert client.kwargs["request_topic"] == \
+            "request:tg:keyword-index:ws:f"
+
+    def test_default_spec_still_requires_topics(self):
+        # Non-optional specs keep the existing contract: a missing topic
+        # is a definition error, surfaced immediately.
+        with pytest.raises(KeyError):
+            make_spec(optional=False).add(
+                make_flow(), MagicMock(), {"topics": {}},
+            )