init

ts/packages/flow/src/gateway/dispatch/manager.ts

@@ -1,14 +1,70 @@
 /**
  * Dispatcher manager — routes requests to backend services via pub/sub.
  *
+ * Maintains a service registry mapping service names to NATS topic pairs.
+ * Applies wire format translation on requests (client → internal) and
+ * reverse translation on responses (internal → client).
+ *
  * Python reference: trustgraph-flow/trustgraph/gateway/dispatch/manager.py
  */
 
 import { NatsBackend, RequestResponse, type PubSubBackend } from "@trustgraph/base";
 import type { GatewayConfig } from "../server.js";
+import { translateRequest, translateResponse } from "./serialize.js";
 
 export type Responder = (response: unknown, complete: boolean) => Promise<void>;
 
+// ---------- Service registry ----------
+
+/**
+ * Flow-scoped request/response services.
+ * These are resolved within a specific flow's interface definitions.
+ * Topic pattern: tg.flow.<name>-request / tg.flow.<name>-response
+ */
+const FLOW_SERVICES: ReadonlyMap<string, { request: string; response: string }> = new Map([
+  ["agent",               { request: "agent-request",               response: "agent-response" }],
+  ["text-completion",     { request: "text-completion-request",     response: "text-completion-response" }],
+  ["prompt",              { request: "prompt-request",              response: "prompt-response" }],
+  ["graph-rag",           { request: "graph-rag-request",           response: "graph-rag-response" }],
+  ["document-rag",        { request: "document-rag-request",        response: "document-rag-response" }],
+  ["embeddings",          { request: "embeddings-request",          response: "embeddings-response" }],
+  ["graph-embeddings",    { request: "graph-embeddings-request",    response: "graph-embeddings-response" }],
+  ["document-embeddings", { request: "doc-embeddings-request",      response: "doc-embeddings-response" }],
+  ["triples",             { request: "triples-request",             response: "triples-response" }],
+]);
+
+/**
+ * Global services (not flow-scoped).
+ * These always use fixed topics regardless of which flow is active.
+ */
+const GLOBAL_SERVICES: ReadonlyMap<string, { request: string; response: string }> = new Map([
+  ["config",                  { request: "config-request",                  response: "config-response" }],
+  ["flow",                    { request: "flow-request",                    response: "flow-response" }],
+  ["librarian",               { request: "librarian-request",               response: "librarian-response" }],
+  ["knowledge",               { request: "knowledge-request",               response: "knowledge-response" }],
+  ["collection-management",   { request: "collection-management-request",   response: "collection-management-response" }],
+]);
+
+/**
+ * Services that support streaming responses (multiple messages per request).
+ * The completion flag is determined by checking for end-of-stream markers.
+ */
+const STREAMING_SERVICES = new Set([
+  "agent",
+  "text-completion",
+  "graph-rag",
+  "document-rag",
+  "triples",
+  "knowledge",
+  "librarian",
+]);
+
+function topicName(name: string): string {
+  return `tg.flow.${name}`;
+}
+
+// ---------- Manager ----------
+
 export class DispatcherManager {
   private pubsub: PubSubBackend;
   private requestors = new Map<string, RequestResponse<unknown, unknown>>();
@@ -18,8 +74,7 @@ export class DispatcherManager {
   }
 
   async start(): Promise<void> {
-    // Pre-create requestors for known global services
-    // Flow-specific requestors are created on demand
+    // Requestors are created on demand when first accessed
   }
 
   async stop(): Promise<void> {
@@ -29,6 +84,8 @@ export class DispatcherManager {
     await this.pubsub.close();
   }
 
+  // ---------- Internal helpers ----------
+
   private async getRequestor(
     requestTopic: string,
     responseTopic: string,
@@ -48,25 +105,71 @@ export class DispatcherManager {
     return rr;
   }
 
+  private resolveGlobalTopics(
+    kind: string,
+  ): { requestTopic: string; responseTopic: string } {
+    const entry = GLOBAL_SERVICES.get(kind);
+    if (entry) {
+      return {
+        requestTopic: topicName(entry.request),
+        responseTopic: topicName(entry.response),
+      };
+    }
+    // Fallback: derive from kind name directly
+    return {
+      requestTopic: topicName(`${kind}-request`),
+      responseTopic: topicName(`${kind}-response`),
+    };
+  }
+
+  private resolveFlowTopics(
+    kind: string,
+  ): { requestTopic: string; responseTopic: string } {
+    const entry = FLOW_SERVICES.get(kind);
+    if (entry) {
+      return {
+        requestTopic: topicName(entry.request),
+        responseTopic: topicName(entry.response),
+      };
+    }
+    // Fallback: derive from kind name directly
+    return {
+      requestTopic: topicName(`${kind}-request`),
+      responseTopic: topicName(`${kind}-response`),
+    };
+  }
+
+  /**
+   * Determine whether a response is the final one in a streaming sequence.
+   * Checks for various end-of-stream markers used by different services.
+   */
+  private isComplete(response: unknown): boolean {
+    if (typeof response !== "object" || response === null) return true;
+    const res = response as Record<string, unknown>;
+    return (
+      !!res.complete ||
+      !!res.endOfStream ||
+      !!res.endOfSession ||
+      !!res.end_of_stream ||
+      !!res.end_of_session ||
+      !!res.eos ||
+      // error responses are always final
+      !!res.error
+    );
+  }
+
+  // ---------- Global service dispatch ----------
+
   async dispatchGlobalService(
     kind: string,
     request: Record<string, unknown>,
   ): Promise<unknown> {
-    const requestTopic = `tg.flow.${kind}-request`;
-    const responseTopic = `tg.flow.${kind}-response`;
+    const { requestTopic, responseTopic } = this.resolveGlobalTopics(kind);
     const rr = await this.getRequestor(requestTopic, responseTopic, `global:${kind}`);
-    return rr.request(request);
-  }
 
-  async dispatchFlowService(
-    flow: string,
-    kind: string,
-    request: Record<string, unknown>,
-  ): Promise<unknown> {
-    const requestTopic = `tg.flow.${kind}-request`;
-    const responseTopic = `tg.flow.${kind}-response`;
-    const rr = await this.getRequestor(requestTopic, responseTopic, `flow:${flow}:${kind}`);
-    return rr.request(request);
+    const translated = translateRequest(kind, request);
+    const response = await rr.request(translated);
+    return translateResponse(kind, response);
   }
 
   async dispatchGlobalServiceStreaming(
@@ -74,37 +177,74 @@ export class DispatcherManager {
     request: Record<string, unknown>,
     responder: Responder,
   ): Promise<void> {
-    const requestTopic = `tg.flow.${kind}-request`;
-    const responseTopic = `tg.flow.${kind}-response`;
+    const { requestTopic, responseTopic } = this.resolveGlobalTopics(kind);
     const rr = await this.getRequestor(requestTopic, responseTopic, `global:${kind}`);
+    const translated = translateRequest(kind, request);
 
-    await rr.request(request, {
+    await rr.request(translated, {
       recipient: async (response) => {
-        const res = response as Record<string, unknown>;
-        const complete = !!res.complete || !!res.endOfStream || !!res.endOfSession;
-        await responder(res, complete);
+        const translatedRes = translateResponse(kind, response);
+        const complete = this.isComplete(translatedRes);
+        await responder(translatedRes, complete);
         return complete;
       },
     });
   }
 
+  // ---------- Flow-scoped service dispatch ----------
+
+  async dispatchFlowService(
+    flow: string,
+    kind: string,
+    request: Record<string, unknown>,
+  ): Promise<unknown> {
+    const { requestTopic, responseTopic } = this.resolveFlowTopics(kind);
+    const rr = await this.getRequestor(
+      requestTopic,
+      responseTopic,
+      `flow:${flow}:${kind}`,
+    );
+
+    const translated = translateRequest(kind, request);
+    const response = await rr.request(translated);
+    return translateResponse(kind, response);
+  }
+
   async dispatchFlowServiceStreaming(
     flow: string,
     kind: string,
     request: Record<string, unknown>,
     responder: Responder,
   ): Promise<void> {
-    const requestTopic = `tg.flow.${kind}-request`;
-    const responseTopic = `tg.flow.${kind}-response`;
-    const rr = await this.getRequestor(requestTopic, responseTopic, `flow:${flow}:${kind}`);
+    const { requestTopic, responseTopic } = this.resolveFlowTopics(kind);
+    const rr = await this.getRequestor(
+      requestTopic,
+      responseTopic,
+      `flow:${flow}:${kind}`,
+    );
+    const translated = translateRequest(kind, request);
 
-    await rr.request(request, {
+    await rr.request(translated, {
       recipient: async (response) => {
-        const res = response as Record<string, unknown>;
-        const complete = !!res.complete || !!res.endOfStream || !!res.endOfSession;
-        await responder(res, complete);
+        const translatedRes = translateResponse(kind, response);
+        const complete = this.isComplete(translatedRes);
+        await responder(translatedRes, complete);
         return complete;
       },
     });
   }
+
+  // ---------- Static introspection ----------
+
+  static get flowServiceNames(): readonly string[] {
+    return [...FLOW_SERVICES.keys()];
+  }
+
+  static get globalServiceNames(): readonly string[] {
+    return [...GLOBAL_SERVICES.keys()];
+  }
+
+  static isStreamingService(kind: string): boolean {
+    return STREAMING_SERVICES.has(kind);
+  }
 }

ts/packages/flow/src/gateway/dispatch/serialize.ts

@@ -0,0 +1,272 @@
+/**
+ * Wire format serializer — translates between the compact client wire format
+ * (used by @trustgraph/client) and the verbose internal format
+ * (used by @trustgraph/base services).
+ *
+ * Client wire format (compact):
+ *   IRI:     { t: "i", i: "<iri>" }
+ *   BLANK:   { t: "b", d: "<id>" }
+ *   LITERAL: { t: "l", v: "<value>", dt?: "<datatype>", ln?: "<language>" }
+ *   TRIPLE:  { t: "t", tr?: { s, p, o, g? } }
+ *
+ * Internal format (verbose):
+ *   IRI:     { type: "IRI", iri: "<iri>" }
+ *   BLANK:   { type: "BLANK", id: "<id>" }
+ *   LITERAL: { type: "LITERAL", value: "<value>", datatype?: "<dt>", language?: "<lang>" }
+ *   TRIPLE:  { type: "TRIPLE", triple: { s, p, o, g? } }
+ *
+ * Python reference: trustgraph-base/trustgraph/messaging/translators/primitives.py
+ */
+
+import type { Term, Triple } from "@trustgraph/base";
+
+// ---------- Client wire format type definitions ----------
+
+interface ClientIriTerm {
+  t: "i";
+  i: string;
+}
+
+interface ClientBlankTerm {
+  t: "b";
+  d: string;
+}
+
+interface ClientLiteralTerm {
+  t: "l";
+  v: string;
+  dt?: string;
+  ln?: string;
+}
+
+interface ClientTripleTerm {
+  t: "t";
+  tr?: ClientTriple;
+}
+
+type ClientTerm = ClientIriTerm | ClientBlankTerm | ClientLiteralTerm | ClientTripleTerm;
+
+interface ClientTriple {
+  s: ClientTerm;
+  p: ClientTerm;
+  o: ClientTerm;
+  g?: string;
+}
+
+// ---------- Client → Internal ----------
+
+export function clientTermToInternal(wire: ClientTerm): Term {
+  switch (wire.t) {
+    case "i":
+      return { type: "IRI", iri: wire.i };
+    case "b":
+      return { type: "BLANK", id: wire.d };
+    case "l":
+      return {
+        type: "LITERAL",
+        value: wire.v,
+        datatype: wire.dt,
+        language: wire.ln,
+      };
+    case "t":
+      return {
+        type: "TRIPLE",
+        triple: wire.tr ? clientTripleToInternal(wire.tr) : undefined!,
+      };
+    default:
+      // Defensive: pass through unknown term types
+      return wire as unknown as Term;
+  }
+}
+
+export function clientTripleToInternal(wire: ClientTriple): Triple {
+  const result: Triple = {
+    s: clientTermToInternal(wire.s),
+    p: clientTermToInternal(wire.p),
+    o: clientTermToInternal(wire.o),
+  };
+  if (wire.g !== undefined) {
+    // In the client wire format, g is a plain string.
+    // In the internal format, g is an optional Term (named graph).
+    // The Python translator treats g as a plain string passthrough,
+    // so we keep it as-is for compatibility.
+    (result as unknown as Record<string, unknown>).g = wire.g;
+  }
+  return result;
+}
+
+// ---------- Internal → Client ----------
+
+export function internalTermToClient(term: Term): ClientTerm {
+  switch (term.type) {
+    case "IRI":
+      return { t: "i", i: term.iri };
+    case "BLANK":
+      return { t: "b", d: term.id };
+    case "LITERAL": {
+      const lit: ClientLiteralTerm = { t: "l", v: term.value };
+      if (term.datatype) lit.dt = term.datatype;
+      if (term.language) lit.ln = term.language;
+      return lit;
+    }
+    case "TRIPLE":
+      return {
+        t: "t",
+        tr: term.triple ? internalTripleToClient(term.triple) : undefined,
+      };
+    default:
+      return term as unknown as ClientTerm;
+  }
+}
+
+export function internalTripleToClient(triple: Triple): ClientTriple {
+  const result: ClientTriple = {
+    s: internalTermToClient(triple.s),
+    p: internalTermToClient(triple.p),
+    o: internalTermToClient(triple.o),
+  };
+  const g = (triple as unknown as Record<string, unknown>).g;
+  if (g !== undefined && g !== null) {
+    if (typeof g === "string") {
+      result.g = g;
+    } else {
+      // If g is a Term, convert it back to client wire format
+      result.g = (g as Record<string, unknown>).iri as string | undefined;
+    }
+  }
+  return result;
+}
+
+// ---------- Deep object translation ----------
+
+/**
+ * Recursively walk an object and translate every Term-shaped value.
+ * A client term is detected by the presence of a `t` property
+ * with value "i", "b", "l", or "t".
+ */
+function isClientTerm(v: unknown): v is ClientTerm {
+  return (
+    typeof v === "object" &&
+    v !== null &&
+    "t" in v &&
+    typeof (v as Record<string, unknown>).t === "string" &&
+    ["i", "b", "l", "t"].includes((v as Record<string, unknown>).t as string)
+  );
+}
+
+/**
+ * An internal term is detected by the presence of a `type` property
+ * with value "IRI", "BLANK", "LITERAL", or "TRIPLE".
+ */
+function isInternalTerm(v: unknown): v is Term {
+  return (
+    typeof v === "object" &&
+    v !== null &&
+    "type" in v &&
+    typeof (v as Record<string, unknown>).type === "string" &&
+    ["IRI", "BLANK", "LITERAL", "TRIPLE"].includes(
+      (v as Record<string, unknown>).type as string,
+    )
+  );
+}
+
+/**
+ * Deep-translate all client Terms in a request body to internal format.
+ * Handles nested objects and arrays.
+ */
+function deepClientToInternal(value: unknown): unknown {
+  if (value === null || value === undefined) return value;
+
+  if (Array.isArray(value)) {
+    return value.map(deepClientToInternal);
+  }
+
+  if (typeof value === "object") {
+    if (isClientTerm(value)) {
+      return clientTermToInternal(value);
+    }
+    const result: Record<string, unknown> = {};
+    for (const [k, v] of Object.entries(value as Record<string, unknown>)) {
+      result[k] = deepClientToInternal(v);
+    }
+    return result;
+  }
+
+  return value;
+}
+
+/**
+ * Deep-translate all internal Terms in a response body to client format.
+ * Handles nested objects and arrays.
+ */
+function deepInternalToClient(value: unknown): unknown {
+  if (value === null || value === undefined) return value;
+
+  if (Array.isArray(value)) {
+    return value.map(deepInternalToClient);
+  }
+
+  if (typeof value === "object") {
+    if (isInternalTerm(value)) {
+      return internalTermToClient(value);
+    }
+    const result: Record<string, unknown> = {};
+    for (const [k, v] of Object.entries(value as Record<string, unknown>)) {
+      result[k] = deepInternalToClient(v);
+    }
+    return result;
+  }
+
+  return value;
+}
+
+// ---------- Services that contain Term fields ----------
+
+/**
+ * Services whose requests contain Term/Triple fields that need translation.
+ * All other services pass through without term translation.
+ */
+const TERM_BEARING_REQUEST_SERVICES = new Set([
+  "triples",
+  "knowledge",
+]);
+
+/**
+ * Services whose responses contain Term/Triple fields that need translation.
+ */
+const TERM_BEARING_RESPONSE_SERVICES = new Set([
+  "triples",
+  "graph-embeddings",
+  "knowledge",
+]);
+
+// ---------- Top-level request / response translators ----------
+
+/**
+ * Translate a client request body to internal format.
+ *
+ * For services that carry Term fields (triples, knowledge), this deep-walks
+ * the request and converts compact → verbose.
+ * All other services pass through unchanged, since their payloads are simple
+ * scalar fields (query strings, limits, etc.).
+ */
+export function translateRequest(service: string, body: unknown): unknown {
+  if (TERM_BEARING_REQUEST_SERVICES.has(service)) {
+    return deepClientToInternal(body);
+  }
+  return body;
+}
+
+/**
+ * Translate an internal response body to client wire format.
+ *
+ * For services that return Term fields (triples, graph-embeddings, knowledge),
+ * this deep-walks the response and converts verbose → compact.
+ * All other services pass through unchanged.
+ */
+export function translateResponse(service: string, response: unknown): unknown {
+  if (TERM_BEARING_RESPONSE_SERVICES.has(service)) {
+    return deepInternalToClient(response);
+  }
+  return response;
+}