mirror of
https://github.com/trustgraph-ai/trustgraph.git
synced 2026-07-07 04:12:10 +02:00
Enforce strict Effect tsgo migrations
This commit is contained in:
elpresidank
parent
64fb23e7d0
commit
f6878d4dd7
49 changed files with 5547 additions and 3250 deletions
|
|
@ -7,13 +7,13 @@
|
|||
* Python reference: trustgraph-flow/trustgraph/gateway/service.py
|
||||
*/
|
||||
|
||||
import Fastify from "fastify";
|
||||
import Fastify, { type FastifyReply } from "fastify";
|
||||
import websocketPlugin from "@fastify/websocket";
|
||||
import { Config, Effect, Exit, Scope } from "effect";
|
||||
import { Clock, Config, Effect, Exit, Random, Scope } from "effect";
|
||||
import * as O from "effect/Option";
|
||||
import * as RpcSerialization from "effect/unstable/rpc/RpcSerialization";
|
||||
import * as EffectSocket from "effect/unstable/socket/Socket";
|
||||
import { optionalStringConfig, registry, toTgError, type PubSubBackend } from "@trustgraph/base";
|
||||
import { messagingLifecycleError, optionalStringConfig, registry, toTgError, type PubSubBackend } from "@trustgraph/base";
|
||||
import { makeDispatcherManager } from "./dispatch/manager.js";
|
||||
import { makeGatewayRpcServer } from "./rpc-server.js";
|
||||
|
||||
|
|
@ -25,187 +25,227 @@ export interface GatewayConfig {
|
|||
pubsub?: PubSubBackend;
|
||||
}
|
||||
|
||||
export async function createGateway(config: GatewayConfig) {
|
||||
export function createGateway(config: GatewayConfig) {
|
||||
const app = Fastify({ logger: true });
|
||||
await app.register(websocketPlugin);
|
||||
|
||||
const dispatcher = makeDispatcherManager(config);
|
||||
await dispatcher.start();
|
||||
const rpcScope = await Effect.runPromise(Scope.make());
|
||||
const rpcServer = await Effect.runPromise(
|
||||
makeGatewayRpcServer(dispatcher).pipe(
|
||||
Effect.provideService(RpcSerialization.RpcSerialization, RpcSerialization.ndjson),
|
||||
Scope.provide(rpcScope),
|
||||
),
|
||||
);
|
||||
|
||||
// Authentication middleware
|
||||
app.addHook("onRequest", async (request, reply) => {
|
||||
if (request.url === "/api/v1/metrics") return;
|
||||
if (request.url.startsWith("/api/v1/rpc")) return; // RPC socket auth via query param
|
||||
|
||||
if (config.secret !== undefined && config.secret.length > 0) {
|
||||
const auth = request.headers.authorization;
|
||||
if (auth === undefined || auth !== `Bearer ${config.secret}`) {
|
||||
reply.code(401).send({ error: "Unauthorized" });
|
||||
}
|
||||
const sendDispatchResult = (reply: FastifyReply, result: unknown): unknown => {
|
||||
const err = (result as Record<string, unknown>)?.error as { type?: string; message?: string } | undefined;
|
||||
if (err !== undefined) {
|
||||
const statusCode = err.type === "not-found" ? 404 : 400;
|
||||
return reply.code(statusCode).send(result);
|
||||
}
|
||||
});
|
||||
|
||||
app.post<{
|
||||
Body: {
|
||||
scope?: string;
|
||||
service?: string;
|
||||
flow?: string;
|
||||
request?: Record<string, unknown>;
|
||||
};
|
||||
}>("/api/v1/workbench/dispatch", async (request, reply) => {
|
||||
const body = request.body;
|
||||
const service = body.service;
|
||||
const payload = body.request;
|
||||
if (service === undefined || service.length === 0 || payload === undefined) {
|
||||
return reply.code(400).send({
|
||||
error: { type: "bad-request", message: "service and request are required" },
|
||||
});
|
||||
}
|
||||
|
||||
try {
|
||||
const result = body.scope === "flow"
|
||||
? await dispatcher.dispatchFlowService(body.flow ?? "default", service, payload)
|
||||
: await dispatcher.dispatchGlobalService(service, payload);
|
||||
const err = (result as Record<string, unknown>)?.error as { type?: string; message?: string } | undefined;
|
||||
if (err !== undefined) {
|
||||
const statusCode = err.type === "not-found" ? 404 : 400;
|
||||
return reply.code(statusCode).send(result);
|
||||
}
|
||||
return result;
|
||||
} catch (err) {
|
||||
reply.code(500).send({ error: toTgError(err) });
|
||||
}
|
||||
});
|
||||
|
||||
// REST endpoint: POST /api/v1/:kind (global services)
|
||||
app.post<{ Params: { kind: string } }>("/api/v1/:kind", async (request, reply) => {
|
||||
const { kind } = request.params;
|
||||
const body = request.body as Record<string, unknown>;
|
||||
|
||||
try {
|
||||
const result = await dispatcher.dispatchGlobalService(kind, body) as Record<string, unknown>;
|
||||
const err = result?.error as { type?: string; message?: string } | undefined;
|
||||
if (err !== undefined) {
|
||||
const statusCode = err.type === "not-found" ? 404 : 400;
|
||||
return reply.code(statusCode).send(result);
|
||||
}
|
||||
return result;
|
||||
} catch (err) {
|
||||
reply.code(500).send({ error: toTgError(err) });
|
||||
}
|
||||
});
|
||||
|
||||
// REST endpoint: POST /api/v1/flow/:flow/service/:kind (flow-scoped services)
|
||||
app.post<{ Params: { flow: string; kind: string } }>(
|
||||
"/api/v1/flow/:flow/service/:kind",
|
||||
async (request, reply) => {
|
||||
const { flow, kind } = request.params;
|
||||
const body = request.body as Record<string, unknown>;
|
||||
|
||||
try {
|
||||
const result = await dispatcher.dispatchFlowService(flow, kind, body) as Record<string, unknown>;
|
||||
const err = result?.error as { type?: string; message?: string } | undefined;
|
||||
if (err !== undefined) {
|
||||
const statusCode = err.type === "not-found" ? 404 : 400;
|
||||
return reply.code(statusCode).send(result);
|
||||
}
|
||||
return result;
|
||||
} catch (err) {
|
||||
reply.code(500).send({ error: toTgError(err) });
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
// REST endpoint: POST /api/v1/flow/:flow/load (trigger document processing)
|
||||
app.post<{ Params: { flow: string } }>(
|
||||
"/api/v1/flow/:flow/load",
|
||||
async (request, reply) => {
|
||||
const { flow } = request.params;
|
||||
const body = request.body as {
|
||||
documentId?: string;
|
||||
user?: string;
|
||||
collection?: string;
|
||||
};
|
||||
|
||||
if (body.documentId === undefined || body.documentId.length === 0) {
|
||||
return reply.code(400).send({
|
||||
error: { type: "bad-request", message: "documentId is required" },
|
||||
});
|
||||
}
|
||||
|
||||
try {
|
||||
const user = body.user ?? "default";
|
||||
const collection = body.collection ?? "default";
|
||||
const documentId = body.documentId;
|
||||
|
||||
// Publish Document message to the decode-input topic
|
||||
const topic = "tg.flow.document";
|
||||
const metadata = {
|
||||
id: `load-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`,
|
||||
root: documentId,
|
||||
user,
|
||||
collection,
|
||||
};
|
||||
|
||||
await dispatcher.publishToTopic(topic, { metadata, documentId });
|
||||
|
||||
return { status: "processing", documentId, flow };
|
||||
} catch (err) {
|
||||
reply.code(500).send({
|
||||
error: toTgError(err),
|
||||
});
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
// Effect RPC WebSocket endpoint: /api/v1/rpc
|
||||
app.get("/api/v1/rpc", { websocket: true }, (socket, request) => {
|
||||
const url = new URL(request.url, `http://${request.headers.host}`);
|
||||
const token = url.searchParams.get("token");
|
||||
if (config.secret !== undefined && config.secret.length > 0 && token !== config.secret) {
|
||||
socket.close(4001, "Unauthorized");
|
||||
return;
|
||||
}
|
||||
|
||||
const program = Effect.scoped(
|
||||
Effect.gen(function* () {
|
||||
const effectSocket = yield* EffectSocket.fromWebSocket(
|
||||
Effect.succeed(socket as unknown as globalThis.WebSocket),
|
||||
{ closeCodeIsError: (code) => code !== 1000 },
|
||||
);
|
||||
yield* rpcServer.onSocket(effectSocket, headersFrom(request.headers));
|
||||
}),
|
||||
);
|
||||
|
||||
Effect.runPromise(program.pipe(Scope.provide(rpcScope))).catch((err) => {
|
||||
console.error("[Gateway] RPC WebSocket error:", err);
|
||||
if (socket.readyState === 1) {
|
||||
socket.close(1011, "Internal server error");
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// Metrics endpoint — returns Prometheus metrics from prom-client
|
||||
app.get("/api/v1/metrics", async (_, reply) => {
|
||||
reply.header("content-type", registry.contentType);
|
||||
return registry.metrics();
|
||||
});
|
||||
|
||||
return {
|
||||
start: () => app.listen({ port: config.port, host: "0.0.0.0" }),
|
||||
stop: async () => {
|
||||
await app.close();
|
||||
await Effect.runPromise(Scope.close(rpcScope, Exit.void));
|
||||
await dispatcher.stop();
|
||||
},
|
||||
return result;
|
||||
};
|
||||
|
||||
const sendDispatchError = (reply: FastifyReply, error: unknown): unknown =>
|
||||
reply.code(500).send({ error: toTgError(error) });
|
||||
|
||||
return Effect.runPromise(
|
||||
Effect.gen(function* () {
|
||||
yield* Effect.tryPromise({
|
||||
try: () => app.register(websocketPlugin),
|
||||
catch: (cause) => messagingLifecycleError("gateway", "register-websocket", cause),
|
||||
});
|
||||
|
||||
yield* Effect.tryPromise({
|
||||
try: () => dispatcher.start(),
|
||||
catch: (cause) => messagingLifecycleError("gateway", "dispatcher-start", cause),
|
||||
});
|
||||
|
||||
const rpcScope = yield* Scope.make();
|
||||
const rpcServer = yield* makeGatewayRpcServer(dispatcher).pipe(
|
||||
Effect.provideService(RpcSerialization.RpcSerialization, RpcSerialization.ndjson),
|
||||
Scope.provide(rpcScope),
|
||||
);
|
||||
|
||||
return { rpcScope, rpcServer };
|
||||
}),
|
||||
).then(({ rpcScope, rpcServer }) => {
|
||||
// Authentication middleware
|
||||
app.addHook("onRequest", (request, reply) => {
|
||||
if (request.url === "/api/v1/metrics") return;
|
||||
if (request.url.startsWith("/api/v1/rpc")) return; // RPC socket auth via query param
|
||||
|
||||
if (config.secret !== undefined && config.secret.length > 0) {
|
||||
const auth = request.headers.authorization;
|
||||
if (auth === undefined || auth !== `Bearer ${config.secret}`) {
|
||||
reply.code(401).send({ error: "Unauthorized" });
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
app.post<{
|
||||
Body: {
|
||||
scope?: string;
|
||||
service?: string;
|
||||
flow?: string;
|
||||
request?: Record<string, unknown>;
|
||||
};
|
||||
}>("/api/v1/workbench/dispatch", (request, reply) => {
|
||||
const body = request.body;
|
||||
const service = body.service;
|
||||
const payload = body.request;
|
||||
if (service === undefined || service.length === 0 || payload === undefined) {
|
||||
return reply.code(400).send({
|
||||
error: { type: "bad-request", message: "service and request are required" },
|
||||
});
|
||||
}
|
||||
|
||||
return Effect.runPromise(
|
||||
Effect.tryPromise({
|
||||
try: () =>
|
||||
body.scope === "flow"
|
||||
? dispatcher.dispatchFlowService(body.flow ?? "default", service, payload)
|
||||
: dispatcher.dispatchGlobalService(service, payload),
|
||||
catch: (cause) => messagingLifecycleError("gateway", "workbench-dispatch", cause),
|
||||
}).pipe(
|
||||
Effect.map((result) => sendDispatchResult(reply, result)),
|
||||
Effect.catch((error) => Effect.succeed(sendDispatchError(reply, error))),
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
// REST endpoint: POST /api/v1/:kind (global services)
|
||||
app.post<{ Params: { kind: string } }>("/api/v1/:kind", (request, reply) => {
|
||||
const { kind } = request.params;
|
||||
const body = request.body as Record<string, unknown>;
|
||||
|
||||
return Effect.runPromise(
|
||||
Effect.tryPromise({
|
||||
try: () => dispatcher.dispatchGlobalService(kind, body),
|
||||
catch: (cause) => messagingLifecycleError("gateway", "global-dispatch", cause),
|
||||
}).pipe(
|
||||
Effect.map((result) => sendDispatchResult(reply, result)),
|
||||
Effect.catch((error) => Effect.succeed(sendDispatchError(reply, error))),
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
// REST endpoint: POST /api/v1/flow/:flow/service/:kind (flow-scoped services)
|
||||
app.post<{ Params: { flow: string; kind: string } }>(
|
||||
"/api/v1/flow/:flow/service/:kind",
|
||||
(request, reply) => {
|
||||
const { flow, kind } = request.params;
|
||||
const body = request.body as Record<string, unknown>;
|
||||
|
||||
return Effect.runPromise(
|
||||
Effect.tryPromise({
|
||||
try: () => dispatcher.dispatchFlowService(flow, kind, body),
|
||||
catch: (cause) => messagingLifecycleError("gateway", "flow-dispatch", cause),
|
||||
}).pipe(
|
||||
Effect.map((result) => sendDispatchResult(reply, result)),
|
||||
Effect.catch((error) => Effect.succeed(sendDispatchError(reply, error))),
|
||||
),
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
// REST endpoint: POST /api/v1/flow/:flow/load (trigger document processing)
|
||||
app.post<{ Params: { flow: string } }>(
|
||||
"/api/v1/flow/:flow/load",
|
||||
(request, reply) => {
|
||||
const { flow } = request.params;
|
||||
const body = request.body as {
|
||||
documentId?: string;
|
||||
user?: string;
|
||||
collection?: string;
|
||||
};
|
||||
|
||||
if (body.documentId === undefined || body.documentId.length === 0) {
|
||||
return reply.code(400).send({
|
||||
error: { type: "bad-request", message: "documentId is required" },
|
||||
});
|
||||
}
|
||||
|
||||
return Effect.runPromise(
|
||||
Effect.gen(function* () {
|
||||
const user = body.user ?? "default";
|
||||
const collection = body.collection ?? "default";
|
||||
const documentId = body.documentId;
|
||||
const timestamp = yield* Clock.currentTimeMillis;
|
||||
const suffix = yield* Random.nextIntBetween(0, 36 ** 6, { halfOpen: true });
|
||||
|
||||
// Publish Document message to the decode-input topic
|
||||
const topic = "tg.flow.document";
|
||||
const metadata = {
|
||||
id: `load-${timestamp}-${suffix.toString(36).padStart(6, "0")}`,
|
||||
root: documentId,
|
||||
user,
|
||||
collection,
|
||||
};
|
||||
|
||||
yield* Effect.tryPromise({
|
||||
try: () => dispatcher.publishToTopic(topic, { metadata, documentId }),
|
||||
catch: (cause) => messagingLifecycleError("gateway", "publish-load", cause),
|
||||
});
|
||||
|
||||
return { status: "processing", documentId, flow };
|
||||
}).pipe(
|
||||
Effect.catch((error) => Effect.succeed(sendDispatchError(reply, error))),
|
||||
),
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
// Effect RPC WebSocket endpoint: /api/v1/rpc
|
||||
app.get("/api/v1/rpc", { websocket: true }, (socket, request) => {
|
||||
const url = new URL(request.url, `http://${request.headers.host}`);
|
||||
const token = url.searchParams.get("token");
|
||||
if (config.secret !== undefined && config.secret.length > 0 && token !== config.secret) {
|
||||
socket.close(4001, "Unauthorized");
|
||||
return;
|
||||
}
|
||||
|
||||
const program = Effect.scoped(
|
||||
Effect.gen(function* () {
|
||||
const effectSocket = yield* EffectSocket.fromWebSocket(
|
||||
Effect.succeed(socket as unknown as globalThis.WebSocket),
|
||||
{ closeCodeIsError: (code) => code !== 1000 },
|
||||
);
|
||||
yield* rpcServer.onSocket(effectSocket, headersFrom(request.headers));
|
||||
}),
|
||||
);
|
||||
|
||||
void Effect.runPromise(program.pipe(Scope.provide(rpcScope))).catch((error) => {
|
||||
void Effect.runPromise(
|
||||
Effect.logError("[Gateway] RPC WebSocket error", { error: toTgError(error).message }).pipe(
|
||||
Effect.flatMap(() =>
|
||||
Effect.sync(() => {
|
||||
if (socket.readyState === 1) {
|
||||
socket.close(1011, "Internal server error");
|
||||
}
|
||||
}),
|
||||
),
|
||||
),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
// Metrics endpoint — returns Prometheus metrics from prom-client
|
||||
app.get("/api/v1/metrics", (_, reply) => {
|
||||
reply.header("content-type", registry.contentType);
|
||||
return registry.metrics();
|
||||
});
|
||||
|
||||
return {
|
||||
start: () => app.listen({ port: config.port, host: "0.0.0.0" }),
|
||||
stop: () =>
|
||||
Effect.runPromise(
|
||||
Effect.gen(function* () {
|
||||
yield* Effect.tryPromise({
|
||||
try: () => app.close(),
|
||||
catch: (cause) => messagingLifecycleError("gateway", "app-close", cause),
|
||||
});
|
||||
yield* Scope.close(rpcScope, Exit.void);
|
||||
yield* Effect.tryPromise({
|
||||
try: () => dispatcher.stop(),
|
||||
catch: (cause) => messagingLifecycleError("gateway", "dispatcher-stop", cause),
|
||||
});
|
||||
}),
|
||||
),
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
function headersFrom(headers: Record<string, string | string[] | number | undefined>): ReadonlyArray<[string, string]> {
|
||||
|
|
@ -217,8 +257,8 @@ function headersFrom(headers: Record<string, string | string[] | number | undefi
|
|||
});
|
||||
}
|
||||
|
||||
export async function run(): Promise<void> {
|
||||
await Effect.runPromise(program);
|
||||
export function run(): Promise<void> {
|
||||
return Effect.runPromise(program);
|
||||
}
|
||||
|
||||
export const loadGatewayConfig = Effect.fn("loadGatewayConfig")(function* () {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue