mirror of
https://github.com/trustgraph-ai/trustgraph.git
synced 2026-07-07 04:12:10 +02:00
Enforce strict Effect tsgo migrations
This commit is contained in:
elpresidank
parent
64fb23e7d0
commit
f6878d4dd7
49 changed files with 5547 additions and 3250 deletions
|
|
@ -8,7 +8,7 @@
|
|||
* Python reference: trustgraph-flow/trustgraph/gateway/dispatch/manager.py
|
||||
*/
|
||||
|
||||
import { Effect, Exit, Scope, SynchronizedRef } from "effect";
|
||||
import { Clock, Effect, Exit, Random, Scope, SynchronizedRef } from "effect";
|
||||
import {
|
||||
loadMessagingRuntimeConfig,
|
||||
makeNatsBackend,
|
||||
|
|
@ -146,13 +146,13 @@ export function makeDispatcherManager(config: GatewayConfig): DispatcherManager
|
|||
const pubsub: PubSubBackend = config.pubsub ?? makeNatsBackend(config.natsUrl ?? "nats://localhost:4222");
|
||||
let runtime: DispatcherRuntime | null = null;
|
||||
|
||||
const start = async (): Promise<void> => {
|
||||
if (runtime !== null) return;
|
||||
const start = (): Promise<void> => {
|
||||
if (runtime !== null) return Promise.resolve();
|
||||
|
||||
runtime = await Effect.runPromise(
|
||||
return Effect.runPromise(
|
||||
Effect.gen(function* () {
|
||||
const scope = yield* Scope.make();
|
||||
return yield* Effect.gen(function* () {
|
||||
const nextRuntime = yield* Effect.gen(function* () {
|
||||
const messagingConfig = yield* loadMessagingRuntimeConfig();
|
||||
const requestors = yield* SynchronizedRef.make<RequestorMap>(new Map());
|
||||
return {
|
||||
|
|
@ -163,62 +163,79 @@ export function makeDispatcherManager(config: GatewayConfig): DispatcherManager
|
|||
}).pipe(
|
||||
Effect.onError((cause) => Scope.close(scope, Exit.failCause(cause))),
|
||||
);
|
||||
runtime = nextRuntime;
|
||||
}),
|
||||
);
|
||||
};
|
||||
|
||||
const stop = async (): Promise<void> => {
|
||||
const current = runtime;
|
||||
runtime = null;
|
||||
const stop = (): Promise<void> =>
|
||||
Effect.runPromise(
|
||||
Effect.gen(function* () {
|
||||
const current = runtime;
|
||||
runtime = null;
|
||||
|
||||
if (current !== null) {
|
||||
await Effect.runPromise(Scope.close(current.scope, Exit.void));
|
||||
}
|
||||
if (current !== null) {
|
||||
yield* Scope.close(current.scope, Exit.void);
|
||||
}
|
||||
|
||||
await pubsub.close();
|
||||
};
|
||||
yield* Effect.tryPromise({
|
||||
try: () => pubsub.close(),
|
||||
catch: (cause) => messagingLifecycleError("gateway-dispatcher", "close-pubsub", cause),
|
||||
});
|
||||
}),
|
||||
);
|
||||
|
||||
// ---------- Internal helpers ----------
|
||||
|
||||
const ensureRuntime = async (): Promise<DispatcherRuntime> => {
|
||||
if (runtime === null) {
|
||||
await start();
|
||||
}
|
||||
if (runtime === null) {
|
||||
throw messagingLifecycleError("gateway-dispatcher", "start", "Dispatcher manager failed to start");
|
||||
}
|
||||
return runtime;
|
||||
};
|
||||
const ensureRuntime = (): Promise<DispatcherRuntime> =>
|
||||
Effect.runPromise(
|
||||
Effect.gen(function* () {
|
||||
if (runtime === null) {
|
||||
yield* Effect.tryPromise({
|
||||
try: () => start(),
|
||||
catch: (cause) => messagingLifecycleError("gateway-dispatcher", "start", cause),
|
||||
});
|
||||
}
|
||||
if (runtime === null) {
|
||||
return yield* messagingLifecycleError("gateway-dispatcher", "start", "Dispatcher manager failed to start");
|
||||
}
|
||||
return runtime;
|
||||
}),
|
||||
);
|
||||
|
||||
const getRequestor = async (
|
||||
const getRequestor = (
|
||||
requestTopic: string,
|
||||
responseTopic: string,
|
||||
key: string,
|
||||
): Promise<EffectRequestResponse<unknown, unknown>> => {
|
||||
const current = await ensureRuntime();
|
||||
): Promise<EffectRequestResponse<unknown, unknown>> =>
|
||||
Effect.runPromise(
|
||||
Effect.gen(function* () {
|
||||
const current = yield* Effect.tryPromise({
|
||||
try: () => ensureRuntime(),
|
||||
catch: (cause) => messagingLifecycleError("gateway-dispatcher", "ensure-runtime", cause),
|
||||
});
|
||||
|
||||
return await Effect.runPromise(
|
||||
SynchronizedRef.modifyEffect(current.requestors, (requestors) => {
|
||||
const cached = requestors.get(key);
|
||||
if (cached !== undefined) {
|
||||
return Effect.succeed([cached, requestors] as const);
|
||||
}
|
||||
return yield* SynchronizedRef.modifyEffect(current.requestors, (requestors) => {
|
||||
const cached = requestors.get(key);
|
||||
if (cached !== undefined) {
|
||||
return Effect.succeed([cached, requestors] as const);
|
||||
}
|
||||
|
||||
return current.factory.make<unknown, unknown>({
|
||||
requestTopic,
|
||||
responseTopic,
|
||||
subscription: `gateway-${key}`,
|
||||
}).pipe(
|
||||
Scope.provide(current.scope),
|
||||
Effect.map((requestor) => {
|
||||
const next = new Map(requestors);
|
||||
next.set(key, requestor);
|
||||
return [requestor, next] as const;
|
||||
}),
|
||||
);
|
||||
return current.factory.make<unknown, unknown>({
|
||||
requestTopic,
|
||||
responseTopic,
|
||||
subscription: `gateway-${key}`,
|
||||
}).pipe(
|
||||
Scope.provide(current.scope),
|
||||
Effect.map((requestor) => {
|
||||
const next = new Map(requestors);
|
||||
next.set(key, requestor);
|
||||
return [requestor, next] as const;
|
||||
}),
|
||||
);
|
||||
});
|
||||
}),
|
||||
);
|
||||
};
|
||||
|
||||
const resolveGlobalTopics = (
|
||||
kind: string,
|
||||
|
|
@ -256,93 +273,107 @@ export function makeDispatcherManager(config: GatewayConfig): DispatcherManager
|
|||
|
||||
// ---------- Global service dispatch ----------
|
||||
|
||||
const dispatchGlobalService = async (
|
||||
const dispatchGlobalService = (
|
||||
kind: string,
|
||||
request: Record<string, unknown>,
|
||||
): Promise<unknown> => {
|
||||
const { requestTopic, responseTopic } = resolveGlobalTopics(kind);
|
||||
const rr = await getRequestor(requestTopic, responseTopic, `global:${kind}`);
|
||||
): Promise<unknown> =>
|
||||
Effect.runPromise(
|
||||
Effect.gen(function* () {
|
||||
const { requestTopic, responseTopic } = resolveGlobalTopics(kind);
|
||||
const rr = yield* Effect.tryPromise({
|
||||
try: () => getRequestor(requestTopic, responseTopic, `global:${kind}`),
|
||||
catch: (cause) => messagingLifecycleError("gateway-dispatcher", "get-requestor", cause),
|
||||
});
|
||||
|
||||
const translated = translateRequest(kind, request);
|
||||
const response = await Effect.runPromise(rr.request(translated));
|
||||
return translateResponse(kind, response);
|
||||
};
|
||||
const translated = translateRequest(kind, request);
|
||||
const response = yield* rr.request(translated);
|
||||
return translateResponse(kind, response);
|
||||
}),
|
||||
);
|
||||
|
||||
const dispatchGlobalServiceStreaming = async (
|
||||
const dispatchGlobalServiceStreaming = (
|
||||
kind: string,
|
||||
request: Record<string, unknown>,
|
||||
responder: Responder,
|
||||
): Promise<void> => {
|
||||
const { requestTopic, responseTopic } = resolveGlobalTopics(kind);
|
||||
const rr = await getRequestor(requestTopic, responseTopic, `global:${kind}`);
|
||||
const translated = translateRequest(kind, request);
|
||||
): Promise<void> =>
|
||||
Effect.runPromise(
|
||||
Effect.gen(function* () {
|
||||
const { requestTopic, responseTopic } = resolveGlobalTopics(kind);
|
||||
const rr = yield* Effect.tryPromise({
|
||||
try: () => getRequestor(requestTopic, responseTopic, `global:${kind}`),
|
||||
catch: (cause) => messagingLifecycleError("gateway-dispatcher", "get-requestor", cause),
|
||||
});
|
||||
const translated = translateRequest(kind, request);
|
||||
|
||||
await Effect.runPromise(
|
||||
rr.request(translated, {
|
||||
recipient: (response) => {
|
||||
const translatedRes = translateResponse(kind, response);
|
||||
const complete = dispatcherManagerIsCompleteResponse(translatedRes);
|
||||
return Effect.tryPromise({
|
||||
try: async () => {
|
||||
await responder(translatedRes, complete);
|
||||
return complete;
|
||||
},
|
||||
catch: (error) => messagingDeliveryError(responseTopic, "stream-responder", error),
|
||||
});
|
||||
},
|
||||
yield* rr.request(translated, {
|
||||
recipient: (response) => {
|
||||
const translatedRes = translateResponse(kind, response);
|
||||
const complete = dispatcherManagerIsCompleteResponse(translatedRes);
|
||||
return Effect.tryPromise({
|
||||
try: () => responder(translatedRes, complete).then(() => complete),
|
||||
catch: (error) => messagingDeliveryError(responseTopic, "stream-responder", error),
|
||||
});
|
||||
},
|
||||
});
|
||||
}),
|
||||
);
|
||||
};
|
||||
|
||||
// ---------- Flow-scoped service dispatch ----------
|
||||
|
||||
const dispatchFlowService = async (
|
||||
const dispatchFlowService = (
|
||||
flow: string,
|
||||
kind: string,
|
||||
request: Record<string, unknown>,
|
||||
): Promise<unknown> => {
|
||||
const { requestTopic, responseTopic } = resolveFlowTopics(kind);
|
||||
const rr = await getRequestor(
|
||||
requestTopic,
|
||||
responseTopic,
|
||||
`flow:${flow}:${kind}`,
|
||||
): Promise<unknown> =>
|
||||
Effect.runPromise(
|
||||
Effect.gen(function* () {
|
||||
const { requestTopic, responseTopic } = resolveFlowTopics(kind);
|
||||
const rr = yield* Effect.tryPromise({
|
||||
try: () => getRequestor(
|
||||
requestTopic,
|
||||
responseTopic,
|
||||
`flow:${flow}:${kind}`,
|
||||
),
|
||||
catch: (cause) => messagingLifecycleError("gateway-dispatcher", "get-requestor", cause),
|
||||
});
|
||||
|
||||
const translated = translateRequest(kind, request);
|
||||
const response = yield* rr.request(translated);
|
||||
return translateResponse(kind, response);
|
||||
}),
|
||||
);
|
||||
|
||||
const translated = translateRequest(kind, request);
|
||||
const response = await Effect.runPromise(rr.request(translated));
|
||||
return translateResponse(kind, response);
|
||||
};
|
||||
|
||||
const dispatchFlowServiceStreaming = async (
|
||||
const dispatchFlowServiceStreaming = (
|
||||
flow: string,
|
||||
kind: string,
|
||||
request: Record<string, unknown>,
|
||||
responder: Responder,
|
||||
): Promise<void> => {
|
||||
const { requestTopic, responseTopic } = resolveFlowTopics(kind);
|
||||
const rr = await getRequestor(
|
||||
requestTopic,
|
||||
responseTopic,
|
||||
`flow:${flow}:${kind}`,
|
||||
);
|
||||
const translated = translateRequest(kind, request);
|
||||
): Promise<void> =>
|
||||
Effect.runPromise(
|
||||
Effect.gen(function* () {
|
||||
const { requestTopic, responseTopic } = resolveFlowTopics(kind);
|
||||
const rr = yield* Effect.tryPromise({
|
||||
try: () => getRequestor(
|
||||
requestTopic,
|
||||
responseTopic,
|
||||
`flow:${flow}:${kind}`,
|
||||
),
|
||||
catch: (cause) => messagingLifecycleError("gateway-dispatcher", "get-requestor", cause),
|
||||
});
|
||||
const translated = translateRequest(kind, request);
|
||||
|
||||
await Effect.runPromise(
|
||||
rr.request(translated, {
|
||||
recipient: (response) => {
|
||||
const translatedRes = translateResponse(kind, response);
|
||||
const complete = dispatcherManagerIsCompleteResponse(translatedRes);
|
||||
return Effect.tryPromise({
|
||||
try: async () => {
|
||||
await responder(translatedRes, complete);
|
||||
return complete;
|
||||
},
|
||||
catch: (error) => messagingDeliveryError(responseTopic, "stream-responder", error),
|
||||
});
|
||||
},
|
||||
yield* rr.request(translated, {
|
||||
recipient: (response) => {
|
||||
const translatedRes = translateResponse(kind, response);
|
||||
const complete = dispatcherManagerIsCompleteResponse(translatedRes);
|
||||
return Effect.tryPromise({
|
||||
try: () => responder(translatedRes, complete).then(() => complete),
|
||||
catch: (error) => messagingDeliveryError(responseTopic, "stream-responder", error),
|
||||
});
|
||||
},
|
||||
});
|
||||
}),
|
||||
);
|
||||
};
|
||||
|
||||
// ---------- Fire-and-forget publish ----------
|
||||
|
||||
|
|
@ -350,12 +381,27 @@ export function makeDispatcherManager(config: GatewayConfig): DispatcherManager
|
|||
* Publish a single message to an arbitrary topic (no request/response).
|
||||
* Used for injecting documents into the processing pipeline.
|
||||
*/
|
||||
const publishToTopic = async (topic: string, message: unknown, id?: string): Promise<void> => {
|
||||
const producer = await pubsub.createProducer<unknown>({ topic });
|
||||
const messageId = id ?? `pub-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
||||
await producer.send(message, { id: messageId });
|
||||
await producer.close();
|
||||
};
|
||||
const publishToTopic = (topic: string, message: unknown, id?: string): Promise<void> =>
|
||||
Effect.runPromise(
|
||||
Effect.gen(function* () {
|
||||
const producer = yield* Effect.tryPromise({
|
||||
try: () => pubsub.createProducer<unknown>({ topic }),
|
||||
catch: (cause) => messagingDeliveryError(topic, "create-producer", cause),
|
||||
});
|
||||
const timestamp = yield* Clock.currentTimeMillis;
|
||||
const suffix = yield* Random.nextIntBetween(0, 36 ** 6, { halfOpen: true });
|
||||
const messageId = id ?? `pub-${timestamp}-${suffix.toString(36).padStart(6, "0")}`;
|
||||
|
||||
yield* Effect.tryPromise({
|
||||
try: () => producer.send(message, { id: messageId }),
|
||||
catch: (cause) => messagingDeliveryError(topic, "send", cause),
|
||||
});
|
||||
yield* Effect.tryPromise({
|
||||
try: () => producer.close(),
|
||||
catch: (cause) => messagingDeliveryError(topic, "close-producer", cause),
|
||||
});
|
||||
}),
|
||||
);
|
||||
|
||||
return {
|
||||
start,
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@
|
|||
*/
|
||||
|
||||
import type { Term, Triple } from "@trustgraph/base";
|
||||
import * as S from "effect/Schema";
|
||||
|
||||
// ---------- Client wire format type definitions ----------
|
||||
|
||||
|
|
@ -46,6 +47,14 @@ interface ClientTripleTerm {
|
|||
|
||||
type ClientTerm = ClientIriTerm | ClientBlankTerm | ClientLiteralTerm | ClientTripleTerm;
|
||||
|
||||
export class DispatchSerializationError extends S.TaggedErrorClass<DispatchSerializationError>()(
|
||||
"DispatchSerializationError",
|
||||
{
|
||||
message: S.String,
|
||||
operation: S.String,
|
||||
},
|
||||
) {}
|
||||
|
||||
interface ClientTriple {
|
||||
s: ClientTerm;
|
||||
p: ClientTerm;
|
||||
|
|
@ -70,7 +79,10 @@ export function clientTermToInternal(wire: ClientTerm): Term {
|
|||
};
|
||||
case "t": {
|
||||
if (wire.tr === undefined) {
|
||||
throw new Error("Client triple term is missing tr");
|
||||
throw DispatchSerializationError.make({
|
||||
operation: "client-term-to-internal",
|
||||
message: "Client triple term is missing tr",
|
||||
});
|
||||
}
|
||||
return {
|
||||
type: "TRIPLE",
|
||||
|
|
|
|||
|
|
@ -34,37 +34,44 @@ export const makeSocketRpcProtocol = Effect.gen(function* () {
|
|||
});
|
||||
|
||||
const writeRaw = yield* socket.writer;
|
||||
const write = (response: RpcMessage.FromServerEncoded) => {
|
||||
try {
|
||||
const encoded = parser.encode(response);
|
||||
if (encoded === undefined) return Effect.void;
|
||||
return Effect.orDie(writeRaw(encoded));
|
||||
} catch (cause) {
|
||||
return Effect.orDie(
|
||||
writeRaw(parser.encode(RpcMessage.ResponseDefectEncoded(cause))!),
|
||||
);
|
||||
}
|
||||
};
|
||||
const encodeDefect = (cause: unknown) =>
|
||||
Effect.sync(() => parser.encode(RpcMessage.ResponseDefectEncoded(cause))!);
|
||||
const write = (response: RpcMessage.FromServerEncoded) =>
|
||||
Effect.sync(() => parser.encode(response)).pipe(
|
||||
Effect.flatMap((encoded) =>
|
||||
encoded === undefined ? Effect.void : Effect.orDie(writeRaw(encoded)),
|
||||
),
|
||||
Effect.catchDefect((cause: unknown) =>
|
||||
encodeDefect(cause).pipe(
|
||||
Effect.flatMap((encoded) => Effect.orDie(writeRaw(encoded))),
|
||||
Effect.orDie,
|
||||
),
|
||||
),
|
||||
);
|
||||
|
||||
clients.set(clientId, { write });
|
||||
clientIds.add(clientId);
|
||||
|
||||
yield* socket.runRaw((data) => {
|
||||
try {
|
||||
const decoded = parser.decode(data) as ReadonlyArray<RpcMessage.FromClientEncoded>;
|
||||
return Effect.forEach(decoded, (message) => {
|
||||
if (message._tag === "Request" && headers !== undefined) {
|
||||
return writeRequest(clientId, {
|
||||
...message,
|
||||
headers: headers.concat(message.headers),
|
||||
});
|
||||
}
|
||||
return writeRequest(clientId, message);
|
||||
}, { discard: true });
|
||||
} catch (cause) {
|
||||
return writeRaw(parser.encode(RpcMessage.ResponseDefectEncoded(cause))!);
|
||||
}
|
||||
}).pipe(
|
||||
yield* socket.runRaw((data) =>
|
||||
Effect.sync(() => parser.decode(data) as ReadonlyArray<RpcMessage.FromClientEncoded>).pipe(
|
||||
Effect.flatMap((decoded) =>
|
||||
Effect.forEach(decoded, (message) => {
|
||||
if (message._tag === "Request" && headers !== undefined) {
|
||||
return writeRequest(clientId, {
|
||||
...message,
|
||||
headers: headers.concat(message.headers),
|
||||
});
|
||||
}
|
||||
return writeRequest(clientId, message);
|
||||
}, { discard: true }),
|
||||
),
|
||||
Effect.catchDefect((cause: unknown) =>
|
||||
encodeDefect(cause).pipe(
|
||||
Effect.flatMap((encoded) => writeRaw(encoded)),
|
||||
),
|
||||
),
|
||||
)
|
||||
).pipe(
|
||||
Effect.catchReason("SocketError", "SocketCloseError", () => Effect.void),
|
||||
Effect.orDie,
|
||||
);
|
||||
|
|
|
|||
|
|
@ -42,7 +42,7 @@ const makeGatewayRpcHandlers = (dispatcher: DispatcherManager) =>
|
|||
Dispatch: (payload) =>
|
||||
Effect.tryPromise({
|
||||
try: () => dispatchOne(dispatcher, payload),
|
||||
catch: (cause) => new DispatchError({ message: errorMessage(cause) }),
|
||||
catch: (cause) => DispatchError.make({ message: errorMessage(cause) }),
|
||||
}),
|
||||
DispatchStream: Effect.fn("GatewayRpc.DispatchStream")(function* (payload) {
|
||||
const context = yield* Effect.context<never>();
|
||||
|
|
@ -52,11 +52,10 @@ const makeGatewayRpcHandlers = (dispatcher: DispatcherManager) =>
|
|||
|
||||
yield* Effect.tryPromise({
|
||||
try: () =>
|
||||
dispatchStream(dispatcher, payload, async (response, complete) => {
|
||||
await runPromise(Queue.offer(queue, new DispatchStreamChunk({ response, complete })));
|
||||
return complete;
|
||||
}),
|
||||
catch: (cause) => new DispatchError({ message: errorMessage(cause) }),
|
||||
dispatchStream(dispatcher, payload, (response, complete) =>
|
||||
runPromise(Queue.offer(queue, DispatchStreamChunk.make({ response, complete }))).then(() => complete),
|
||||
),
|
||||
catch: (cause) => DispatchError.make({ message: errorMessage(cause) }),
|
||||
}).pipe(
|
||||
Effect.flatMap(() => Queue.end(queue)),
|
||||
Effect.catch((error) => Queue.fail(queue, error)),
|
||||
|
|
@ -82,26 +81,24 @@ function dispatchOne(
|
|||
return dispatcher.dispatchGlobalService(payload.service, payload.request);
|
||||
}
|
||||
|
||||
async function dispatchStream(
|
||||
function dispatchStream(
|
||||
dispatcher: DispatcherManager,
|
||||
payload: DispatchPayload,
|
||||
responder: (response: unknown, complete: boolean) => Promise<boolean>,
|
||||
): Promise<void> {
|
||||
const send = async (response: unknown, complete: boolean) => {
|
||||
await responder(response, complete);
|
||||
};
|
||||
const send = (response: unknown, complete: boolean): Promise<void> =>
|
||||
responder(response, complete).then(() => undefined);
|
||||
|
||||
if (payload.scope === "flow") {
|
||||
await dispatcher.dispatchFlowServiceStreaming(
|
||||
return dispatcher.dispatchFlowServiceStreaming(
|
||||
payload.flow ?? "default",
|
||||
payload.service,
|
||||
payload.request,
|
||||
send,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
await dispatcher.dispatchGlobalServiceStreaming(
|
||||
return dispatcher.dispatchGlobalServiceStreaming(
|
||||
payload.service,
|
||||
payload.request,
|
||||
send,
|
||||
|
|
|
|||
|
|
@ -7,13 +7,13 @@
|
|||
* Python reference: trustgraph-flow/trustgraph/gateway/service.py
|
||||
*/
|
||||
|
||||
import Fastify from "fastify";
|
||||
import Fastify, { type FastifyReply } from "fastify";
|
||||
import websocketPlugin from "@fastify/websocket";
|
||||
import { Config, Effect, Exit, Scope } from "effect";
|
||||
import { Clock, Config, Effect, Exit, Random, Scope } from "effect";
|
||||
import * as O from "effect/Option";
|
||||
import * as RpcSerialization from "effect/unstable/rpc/RpcSerialization";
|
||||
import * as EffectSocket from "effect/unstable/socket/Socket";
|
||||
import { optionalStringConfig, registry, toTgError, type PubSubBackend } from "@trustgraph/base";
|
||||
import { messagingLifecycleError, optionalStringConfig, registry, toTgError, type PubSubBackend } from "@trustgraph/base";
|
||||
import { makeDispatcherManager } from "./dispatch/manager.js";
|
||||
import { makeGatewayRpcServer } from "./rpc-server.js";
|
||||
|
||||
|
|
@ -25,187 +25,227 @@ export interface GatewayConfig {
|
|||
pubsub?: PubSubBackend;
|
||||
}
|
||||
|
||||
export async function createGateway(config: GatewayConfig) {
|
||||
export function createGateway(config: GatewayConfig) {
|
||||
const app = Fastify({ logger: true });
|
||||
await app.register(websocketPlugin);
|
||||
|
||||
const dispatcher = makeDispatcherManager(config);
|
||||
await dispatcher.start();
|
||||
const rpcScope = await Effect.runPromise(Scope.make());
|
||||
const rpcServer = await Effect.runPromise(
|
||||
makeGatewayRpcServer(dispatcher).pipe(
|
||||
Effect.provideService(RpcSerialization.RpcSerialization, RpcSerialization.ndjson),
|
||||
Scope.provide(rpcScope),
|
||||
),
|
||||
);
|
||||
|
||||
// Authentication middleware
|
||||
app.addHook("onRequest", async (request, reply) => {
|
||||
if (request.url === "/api/v1/metrics") return;
|
||||
if (request.url.startsWith("/api/v1/rpc")) return; // RPC socket auth via query param
|
||||
|
||||
if (config.secret !== undefined && config.secret.length > 0) {
|
||||
const auth = request.headers.authorization;
|
||||
if (auth === undefined || auth !== `Bearer ${config.secret}`) {
|
||||
reply.code(401).send({ error: "Unauthorized" });
|
||||
}
|
||||
const sendDispatchResult = (reply: FastifyReply, result: unknown): unknown => {
|
||||
const err = (result as Record<string, unknown>)?.error as { type?: string; message?: string } | undefined;
|
||||
if (err !== undefined) {
|
||||
const statusCode = err.type === "not-found" ? 404 : 400;
|
||||
return reply.code(statusCode).send(result);
|
||||
}
|
||||
});
|
||||
|
||||
app.post<{
|
||||
Body: {
|
||||
scope?: string;
|
||||
service?: string;
|
||||
flow?: string;
|
||||
request?: Record<string, unknown>;
|
||||
};
|
||||
}>("/api/v1/workbench/dispatch", async (request, reply) => {
|
||||
const body = request.body;
|
||||
const service = body.service;
|
||||
const payload = body.request;
|
||||
if (service === undefined || service.length === 0 || payload === undefined) {
|
||||
return reply.code(400).send({
|
||||
error: { type: "bad-request", message: "service and request are required" },
|
||||
});
|
||||
}
|
||||
|
||||
try {
|
||||
const result = body.scope === "flow"
|
||||
? await dispatcher.dispatchFlowService(body.flow ?? "default", service, payload)
|
||||
: await dispatcher.dispatchGlobalService(service, payload);
|
||||
const err = (result as Record<string, unknown>)?.error as { type?: string; message?: string } | undefined;
|
||||
if (err !== undefined) {
|
||||
const statusCode = err.type === "not-found" ? 404 : 400;
|
||||
return reply.code(statusCode).send(result);
|
||||
}
|
||||
return result;
|
||||
} catch (err) {
|
||||
reply.code(500).send({ error: toTgError(err) });
|
||||
}
|
||||
});
|
||||
|
||||
// REST endpoint: POST /api/v1/:kind (global services)
|
||||
app.post<{ Params: { kind: string } }>("/api/v1/:kind", async (request, reply) => {
|
||||
const { kind } = request.params;
|
||||
const body = request.body as Record<string, unknown>;
|
||||
|
||||
try {
|
||||
const result = await dispatcher.dispatchGlobalService(kind, body) as Record<string, unknown>;
|
||||
const err = result?.error as { type?: string; message?: string } | undefined;
|
||||
if (err !== undefined) {
|
||||
const statusCode = err.type === "not-found" ? 404 : 400;
|
||||
return reply.code(statusCode).send(result);
|
||||
}
|
||||
return result;
|
||||
} catch (err) {
|
||||
reply.code(500).send({ error: toTgError(err) });
|
||||
}
|
||||
});
|
||||
|
||||
// REST endpoint: POST /api/v1/flow/:flow/service/:kind (flow-scoped services)
|
||||
app.post<{ Params: { flow: string; kind: string } }>(
|
||||
"/api/v1/flow/:flow/service/:kind",
|
||||
async (request, reply) => {
|
||||
const { flow, kind } = request.params;
|
||||
const body = request.body as Record<string, unknown>;
|
||||
|
||||
try {
|
||||
const result = await dispatcher.dispatchFlowService(flow, kind, body) as Record<string, unknown>;
|
||||
const err = result?.error as { type?: string; message?: string } | undefined;
|
||||
if (err !== undefined) {
|
||||
const statusCode = err.type === "not-found" ? 404 : 400;
|
||||
return reply.code(statusCode).send(result);
|
||||
}
|
||||
return result;
|
||||
} catch (err) {
|
||||
reply.code(500).send({ error: toTgError(err) });
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
// REST endpoint: POST /api/v1/flow/:flow/load (trigger document processing)
|
||||
app.post<{ Params: { flow: string } }>(
|
||||
"/api/v1/flow/:flow/load",
|
||||
async (request, reply) => {
|
||||
const { flow } = request.params;
|
||||
const body = request.body as {
|
||||
documentId?: string;
|
||||
user?: string;
|
||||
collection?: string;
|
||||
};
|
||||
|
||||
if (body.documentId === undefined || body.documentId.length === 0) {
|
||||
return reply.code(400).send({
|
||||
error: { type: "bad-request", message: "documentId is required" },
|
||||
});
|
||||
}
|
||||
|
||||
try {
|
||||
const user = body.user ?? "default";
|
||||
const collection = body.collection ?? "default";
|
||||
const documentId = body.documentId;
|
||||
|
||||
// Publish Document message to the decode-input topic
|
||||
const topic = "tg.flow.document";
|
||||
const metadata = {
|
||||
id: `load-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`,
|
||||
root: documentId,
|
||||
user,
|
||||
collection,
|
||||
};
|
||||
|
||||
await dispatcher.publishToTopic(topic, { metadata, documentId });
|
||||
|
||||
return { status: "processing", documentId, flow };
|
||||
} catch (err) {
|
||||
reply.code(500).send({
|
||||
error: toTgError(err),
|
||||
});
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
// Effect RPC WebSocket endpoint: /api/v1/rpc
|
||||
app.get("/api/v1/rpc", { websocket: true }, (socket, request) => {
|
||||
const url = new URL(request.url, `http://${request.headers.host}`);
|
||||
const token = url.searchParams.get("token");
|
||||
if (config.secret !== undefined && config.secret.length > 0 && token !== config.secret) {
|
||||
socket.close(4001, "Unauthorized");
|
||||
return;
|
||||
}
|
||||
|
||||
const program = Effect.scoped(
|
||||
Effect.gen(function* () {
|
||||
const effectSocket = yield* EffectSocket.fromWebSocket(
|
||||
Effect.succeed(socket as unknown as globalThis.WebSocket),
|
||||
{ closeCodeIsError: (code) => code !== 1000 },
|
||||
);
|
||||
yield* rpcServer.onSocket(effectSocket, headersFrom(request.headers));
|
||||
}),
|
||||
);
|
||||
|
||||
Effect.runPromise(program.pipe(Scope.provide(rpcScope))).catch((err) => {
|
||||
console.error("[Gateway] RPC WebSocket error:", err);
|
||||
if (socket.readyState === 1) {
|
||||
socket.close(1011, "Internal server error");
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// Metrics endpoint — returns Prometheus metrics from prom-client
|
||||
app.get("/api/v1/metrics", async (_, reply) => {
|
||||
reply.header("content-type", registry.contentType);
|
||||
return registry.metrics();
|
||||
});
|
||||
|
||||
return {
|
||||
start: () => app.listen({ port: config.port, host: "0.0.0.0" }),
|
||||
stop: async () => {
|
||||
await app.close();
|
||||
await Effect.runPromise(Scope.close(rpcScope, Exit.void));
|
||||
await dispatcher.stop();
|
||||
},
|
||||
return result;
|
||||
};
|
||||
|
||||
const sendDispatchError = (reply: FastifyReply, error: unknown): unknown =>
|
||||
reply.code(500).send({ error: toTgError(error) });
|
||||
|
||||
return Effect.runPromise(
|
||||
Effect.gen(function* () {
|
||||
yield* Effect.tryPromise({
|
||||
try: () => app.register(websocketPlugin),
|
||||
catch: (cause) => messagingLifecycleError("gateway", "register-websocket", cause),
|
||||
});
|
||||
|
||||
yield* Effect.tryPromise({
|
||||
try: () => dispatcher.start(),
|
||||
catch: (cause) => messagingLifecycleError("gateway", "dispatcher-start", cause),
|
||||
});
|
||||
|
||||
const rpcScope = yield* Scope.make();
|
||||
const rpcServer = yield* makeGatewayRpcServer(dispatcher).pipe(
|
||||
Effect.provideService(RpcSerialization.RpcSerialization, RpcSerialization.ndjson),
|
||||
Scope.provide(rpcScope),
|
||||
);
|
||||
|
||||
return { rpcScope, rpcServer };
|
||||
}),
|
||||
).then(({ rpcScope, rpcServer }) => {
|
||||
// Authentication middleware
|
||||
app.addHook("onRequest", (request, reply) => {
|
||||
if (request.url === "/api/v1/metrics") return;
|
||||
if (request.url.startsWith("/api/v1/rpc")) return; // RPC socket auth via query param
|
||||
|
||||
if (config.secret !== undefined && config.secret.length > 0) {
|
||||
const auth = request.headers.authorization;
|
||||
if (auth === undefined || auth !== `Bearer ${config.secret}`) {
|
||||
reply.code(401).send({ error: "Unauthorized" });
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
app.post<{
|
||||
Body: {
|
||||
scope?: string;
|
||||
service?: string;
|
||||
flow?: string;
|
||||
request?: Record<string, unknown>;
|
||||
};
|
||||
}>("/api/v1/workbench/dispatch", (request, reply) => {
|
||||
const body = request.body;
|
||||
const service = body.service;
|
||||
const payload = body.request;
|
||||
if (service === undefined || service.length === 0 || payload === undefined) {
|
||||
return reply.code(400).send({
|
||||
error: { type: "bad-request", message: "service and request are required" },
|
||||
});
|
||||
}
|
||||
|
||||
return Effect.runPromise(
|
||||
Effect.tryPromise({
|
||||
try: () =>
|
||||
body.scope === "flow"
|
||||
? dispatcher.dispatchFlowService(body.flow ?? "default", service, payload)
|
||||
: dispatcher.dispatchGlobalService(service, payload),
|
||||
catch: (cause) => messagingLifecycleError("gateway", "workbench-dispatch", cause),
|
||||
}).pipe(
|
||||
Effect.map((result) => sendDispatchResult(reply, result)),
|
||||
Effect.catch((error) => Effect.succeed(sendDispatchError(reply, error))),
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
// REST endpoint: POST /api/v1/:kind (global services)
|
||||
app.post<{ Params: { kind: string } }>("/api/v1/:kind", (request, reply) => {
|
||||
const { kind } = request.params;
|
||||
const body = request.body as Record<string, unknown>;
|
||||
|
||||
return Effect.runPromise(
|
||||
Effect.tryPromise({
|
||||
try: () => dispatcher.dispatchGlobalService(kind, body),
|
||||
catch: (cause) => messagingLifecycleError("gateway", "global-dispatch", cause),
|
||||
}).pipe(
|
||||
Effect.map((result) => sendDispatchResult(reply, result)),
|
||||
Effect.catch((error) => Effect.succeed(sendDispatchError(reply, error))),
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
// REST endpoint: POST /api/v1/flow/:flow/service/:kind (flow-scoped services)
|
||||
app.post<{ Params: { flow: string; kind: string } }>(
|
||||
"/api/v1/flow/:flow/service/:kind",
|
||||
(request, reply) => {
|
||||
const { flow, kind } = request.params;
|
||||
const body = request.body as Record<string, unknown>;
|
||||
|
||||
return Effect.runPromise(
|
||||
Effect.tryPromise({
|
||||
try: () => dispatcher.dispatchFlowService(flow, kind, body),
|
||||
catch: (cause) => messagingLifecycleError("gateway", "flow-dispatch", cause),
|
||||
}).pipe(
|
||||
Effect.map((result) => sendDispatchResult(reply, result)),
|
||||
Effect.catch((error) => Effect.succeed(sendDispatchError(reply, error))),
|
||||
),
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
// REST endpoint: POST /api/v1/flow/:flow/load (trigger document processing)
|
||||
app.post<{ Params: { flow: string } }>(
|
||||
"/api/v1/flow/:flow/load",
|
||||
(request, reply) => {
|
||||
const { flow } = request.params;
|
||||
const body = request.body as {
|
||||
documentId?: string;
|
||||
user?: string;
|
||||
collection?: string;
|
||||
};
|
||||
|
||||
if (body.documentId === undefined || body.documentId.length === 0) {
|
||||
return reply.code(400).send({
|
||||
error: { type: "bad-request", message: "documentId is required" },
|
||||
});
|
||||
}
|
||||
|
||||
return Effect.runPromise(
|
||||
Effect.gen(function* () {
|
||||
const user = body.user ?? "default";
|
||||
const collection = body.collection ?? "default";
|
||||
const documentId = body.documentId;
|
||||
const timestamp = yield* Clock.currentTimeMillis;
|
||||
const suffix = yield* Random.nextIntBetween(0, 36 ** 6, { halfOpen: true });
|
||||
|
||||
// Publish Document message to the decode-input topic
|
||||
const topic = "tg.flow.document";
|
||||
const metadata = {
|
||||
id: `load-${timestamp}-${suffix.toString(36).padStart(6, "0")}`,
|
||||
root: documentId,
|
||||
user,
|
||||
collection,
|
||||
};
|
||||
|
||||
yield* Effect.tryPromise({
|
||||
try: () => dispatcher.publishToTopic(topic, { metadata, documentId }),
|
||||
catch: (cause) => messagingLifecycleError("gateway", "publish-load", cause),
|
||||
});
|
||||
|
||||
return { status: "processing", documentId, flow };
|
||||
}).pipe(
|
||||
Effect.catch((error) => Effect.succeed(sendDispatchError(reply, error))),
|
||||
),
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
// Effect RPC WebSocket endpoint: /api/v1/rpc
|
||||
app.get("/api/v1/rpc", { websocket: true }, (socket, request) => {
|
||||
const url = new URL(request.url, `http://${request.headers.host}`);
|
||||
const token = url.searchParams.get("token");
|
||||
if (config.secret !== undefined && config.secret.length > 0 && token !== config.secret) {
|
||||
socket.close(4001, "Unauthorized");
|
||||
return;
|
||||
}
|
||||
|
||||
const program = Effect.scoped(
|
||||
Effect.gen(function* () {
|
||||
const effectSocket = yield* EffectSocket.fromWebSocket(
|
||||
Effect.succeed(socket as unknown as globalThis.WebSocket),
|
||||
{ closeCodeIsError: (code) => code !== 1000 },
|
||||
);
|
||||
yield* rpcServer.onSocket(effectSocket, headersFrom(request.headers));
|
||||
}),
|
||||
);
|
||||
|
||||
void Effect.runPromise(program.pipe(Scope.provide(rpcScope))).catch((error) => {
|
||||
void Effect.runPromise(
|
||||
Effect.logError("[Gateway] RPC WebSocket error", { error: toTgError(error).message }).pipe(
|
||||
Effect.flatMap(() =>
|
||||
Effect.sync(() => {
|
||||
if (socket.readyState === 1) {
|
||||
socket.close(1011, "Internal server error");
|
||||
}
|
||||
}),
|
||||
),
|
||||
),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
// Metrics endpoint — returns Prometheus metrics from prom-client
|
||||
app.get("/api/v1/metrics", (_, reply) => {
|
||||
reply.header("content-type", registry.contentType);
|
||||
return registry.metrics();
|
||||
});
|
||||
|
||||
return {
|
||||
start: () => app.listen({ port: config.port, host: "0.0.0.0" }),
|
||||
stop: () =>
|
||||
Effect.runPromise(
|
||||
Effect.gen(function* () {
|
||||
yield* Effect.tryPromise({
|
||||
try: () => app.close(),
|
||||
catch: (cause) => messagingLifecycleError("gateway", "app-close", cause),
|
||||
});
|
||||
yield* Scope.close(rpcScope, Exit.void);
|
||||
yield* Effect.tryPromise({
|
||||
try: () => dispatcher.stop(),
|
||||
catch: (cause) => messagingLifecycleError("gateway", "dispatcher-stop", cause),
|
||||
});
|
||||
}),
|
||||
),
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
function headersFrom(headers: Record<string, string | string[] | number | undefined>): ReadonlyArray<[string, string]> {
|
||||
|
|
@ -217,8 +257,8 @@ function headersFrom(headers: Record<string, string | string[] | number | undefi
|
|||
});
|
||||
}
|
||||
|
||||
export async function run(): Promise<void> {
|
||||
await Effect.runPromise(program);
|
||||
export function run(): Promise<void> {
|
||||
return Effect.runPromise(program);
|
||||
}
|
||||
|
||||
export const loadGatewayConfig = Effect.fn("loadGatewayConfig")(function* () {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue