mirror of
https://github.com/trustgraph-ai/trustgraph.git
synced 2026-07-04 19:02:11 +02:00
Advance TS port Effect workbench
This commit is contained in:
elpresidank
parent
92dae8c374
commit
3515106670
116 changed files with 12286 additions and 9584 deletions
|
|
@ -15,83 +15,112 @@ import {
|
|||
RequestResponseSpec,
|
||||
type ProcessorConfig,
|
||||
type FlowContext,
|
||||
type FlowResourceNotFoundError,
|
||||
type MessagingDeliveryError,
|
||||
type MessagingTimeoutError,
|
||||
type EntityContexts,
|
||||
type EmbeddingsRequest,
|
||||
type EmbeddingsResponse,
|
||||
type Spec,
|
||||
} from "@trustgraph/base";
|
||||
import { makeProcessorProgram } from "@trustgraph/base";
|
||||
import { QdrantGraphEmbeddingsStore } from "./qdrant-graph.js";
|
||||
import { makeFlowProcessorProgram } from "@trustgraph/base";
|
||||
import { Effect } from "effect";
|
||||
import {
|
||||
QdrantGraphEmbeddingsStoreLive,
|
||||
QdrantGraphEmbeddingsStoreService,
|
||||
makeQdrantGraphEmbeddingsStoreService,
|
||||
type QdrantGraphEmbeddingsConfig,
|
||||
type QdrantGraphEmbeddingsStoreError,
|
||||
} from "./qdrant-graph.js";
|
||||
|
||||
export class GraphEmbeddingsStoreService extends FlowProcessor {
|
||||
private store: QdrantGraphEmbeddingsStore;
|
||||
type GraphEmbeddingsStoreRequirements = QdrantGraphEmbeddingsStoreService;
|
||||
type GraphEmbeddingsStoreError =
|
||||
| FlowResourceNotFoundError
|
||||
| MessagingDeliveryError
|
||||
| MessagingTimeoutError
|
||||
| QdrantGraphEmbeddingsStoreError;
|
||||
|
||||
const onGraphEmbeddingsStoreMessage = Effect.fn("GraphEmbeddingsStoreService.onMessage")(function* (
|
||||
msg: EntityContexts,
|
||||
_properties: Record<string, string>,
|
||||
flowCtx: FlowContext<GraphEmbeddingsStoreRequirements>,
|
||||
): Effect.fn.Return<void, GraphEmbeddingsStoreError, GraphEmbeddingsStoreRequirements> {
|
||||
if (msg.entities.length === 0) return;
|
||||
|
||||
const embeddingsClient =
|
||||
yield* flowCtx.flow.requestorEffect<EmbeddingsRequest, EmbeddingsResponse>("embeddings-client");
|
||||
|
||||
const user = msg.metadata?.user ?? "default";
|
||||
const collection = msg.metadata?.collection ?? "default";
|
||||
const texts = msg.entities.map((entity) => entity.context);
|
||||
|
||||
const embResponse = yield* embeddingsClient.request({ text: texts });
|
||||
if (embResponse.error !== undefined) {
|
||||
yield* Effect.logError("[GraphEmbeddingsStore] Embeddings error", {
|
||||
error: embResponse.error.message,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const entities = msg.entities.map((entity, index) => ({
|
||||
entity: entity.entity,
|
||||
vector: embResponse.vectors[index],
|
||||
chunkId: entity.chunkId,
|
||||
}));
|
||||
const store = yield* QdrantGraphEmbeddingsStoreService;
|
||||
|
||||
yield* store.store({ user, collection, entities });
|
||||
|
||||
yield* Effect.log(
|
||||
`[GraphEmbeddingsStore] Stored ${entities.length} embeddings for ${user}/${collection}`,
|
||||
);
|
||||
});
|
||||
|
||||
export const makeGraphEmbeddingsStoreSpecs = (): ReadonlyArray<Spec<GraphEmbeddingsStoreRequirements>> => [
|
||||
new ConsumerSpec<EntityContexts, GraphEmbeddingsStoreError, GraphEmbeddingsStoreRequirements>(
|
||||
"store-graph-embeddings-input",
|
||||
onGraphEmbeddingsStoreMessage,
|
||||
),
|
||||
new RequestResponseSpec<EmbeddingsRequest, EmbeddingsResponse>(
|
||||
"embeddings-client",
|
||||
"embeddings-request",
|
||||
"embeddings-response",
|
||||
),
|
||||
];
|
||||
|
||||
export class GraphEmbeddingsStoreService extends FlowProcessor<GraphEmbeddingsStoreRequirements> {
|
||||
private readonly store = makeQdrantGraphEmbeddingsStoreService();
|
||||
|
||||
constructor(config: ProcessorConfig) {
|
||||
super(config);
|
||||
this.store = new QdrantGraphEmbeddingsStore();
|
||||
|
||||
this.registerSpecification(
|
||||
ConsumerSpec.fromPromise<EntityContexts>(
|
||||
"store-graph-embeddings-input",
|
||||
this.onMessage.bind(this),
|
||||
),
|
||||
);
|
||||
this.registerSpecification(
|
||||
new RequestResponseSpec<EmbeddingsRequest, EmbeddingsResponse>(
|
||||
"embeddings-client",
|
||||
"embeddings-request",
|
||||
"embeddings-response",
|
||||
),
|
||||
);
|
||||
for (const spec of makeGraphEmbeddingsStoreSpecs()) {
|
||||
this.registerSpecification(spec);
|
||||
}
|
||||
|
||||
console.log("[GraphEmbeddingsStore] Service initialized");
|
||||
}
|
||||
|
||||
private async onMessage(
|
||||
msg: EntityContexts,
|
||||
_properties: Record<string, string>,
|
||||
flowCtx: FlowContext,
|
||||
): Promise<void> {
|
||||
if (msg.entities.length === 0) return;
|
||||
|
||||
const embeddingsClient =
|
||||
flowCtx.flow.requestor<EmbeddingsRequest, EmbeddingsResponse>("embeddings-client");
|
||||
|
||||
const user = msg.metadata?.user ?? "default";
|
||||
const collection = msg.metadata?.collection ?? "default";
|
||||
|
||||
// Get text contexts for vectorization
|
||||
const texts = msg.entities.map((e) => e.context);
|
||||
|
||||
// Call embeddings service
|
||||
const embResponse = await embeddingsClient.request({ text: texts });
|
||||
if (embResponse.error !== undefined) {
|
||||
console.error(
|
||||
"[GraphEmbeddingsStore] Embeddings error:",
|
||||
embResponse.error.message,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// Store entity+vector pairs
|
||||
const entities = msg.entities.map((e, i) => ({
|
||||
entity: e.entity,
|
||||
vector: embResponse.vectors[i],
|
||||
chunkId: e.chunkId,
|
||||
}));
|
||||
|
||||
await this.store.store({ user, collection, entities });
|
||||
|
||||
console.log(
|
||||
`[GraphEmbeddingsStore] Stored ${entities.length} embeddings for ${user}/${collection}`,
|
||||
override startEffect() {
|
||||
return super.startEffect().pipe(
|
||||
Effect.provideService(
|
||||
QdrantGraphEmbeddingsStoreService,
|
||||
QdrantGraphEmbeddingsStoreService.of(this.store),
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export const program = makeProcessorProgram({
|
||||
export const program = makeFlowProcessorProgram<
|
||||
ProcessorConfig & QdrantGraphEmbeddingsConfig,
|
||||
never,
|
||||
GraphEmbeddingsStoreRequirements
|
||||
>({
|
||||
id: "graph-embeddings-store",
|
||||
make: (config) => new GraphEmbeddingsStoreService(config),
|
||||
specs: () => makeGraphEmbeddingsStoreSpecs(),
|
||||
layer: (config) => QdrantGraphEmbeddingsStoreLive(config),
|
||||
});
|
||||
|
||||
export async function run(): Promise<void> {
|
||||
await GraphEmbeddingsStoreService.launch("graph-embeddings-store");
|
||||
await Effect.runPromise(program);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,7 +9,9 @@
|
|||
*/
|
||||
|
||||
import { QdrantClient } from "@qdrant/js-client-rest";
|
||||
import type { Term } from "@trustgraph/base";
|
||||
import { errorMessage, type Term } from "@trustgraph/base";
|
||||
import { Context, Effect, Layer } from "effect";
|
||||
import * as S from "effect/Schema";
|
||||
|
||||
export interface QdrantGraphEmbeddingsConfig {
|
||||
url?: string;
|
||||
|
|
@ -127,3 +129,67 @@ export class QdrantGraphEmbeddingsStore {
|
|||
);
|
||||
}
|
||||
}
|
||||
|
||||
export class QdrantGraphEmbeddingsStoreError extends S.TaggedErrorClass<QdrantGraphEmbeddingsStoreError>()(
|
||||
"QdrantGraphEmbeddingsStoreError",
|
||||
{
|
||||
message: S.String,
|
||||
operation: S.String,
|
||||
cause: S.DefectWithStack,
|
||||
},
|
||||
) {}
|
||||
|
||||
export interface QdrantGraphEmbeddingsStoreServiceShape {
|
||||
readonly store: (
|
||||
message: GraphEmbeddingsMessage,
|
||||
) => Effect.Effect<void, QdrantGraphEmbeddingsStoreError>;
|
||||
readonly deleteCollection: (
|
||||
user: string,
|
||||
collection: string,
|
||||
) => Effect.Effect<void, QdrantGraphEmbeddingsStoreError>;
|
||||
}
|
||||
|
||||
export class QdrantGraphEmbeddingsStoreService extends Context.Service<
|
||||
QdrantGraphEmbeddingsStoreService,
|
||||
QdrantGraphEmbeddingsStoreServiceShape
|
||||
>()(
|
||||
"@trustgraph/flow/storage/embeddings/qdrant-graph/QdrantGraphEmbeddingsStoreService",
|
||||
) {}
|
||||
|
||||
const qdrantGraphEmbeddingsStoreError = (operation: string, cause: unknown) =>
|
||||
new QdrantGraphEmbeddingsStoreError({
|
||||
operation,
|
||||
message: errorMessage(cause),
|
||||
cause,
|
||||
});
|
||||
|
||||
export const makeQdrantGraphEmbeddingsStoreService = (
|
||||
config: QdrantGraphEmbeddingsConfig = {},
|
||||
): QdrantGraphEmbeddingsStoreServiceShape => {
|
||||
const store = new QdrantGraphEmbeddingsStore(config);
|
||||
return {
|
||||
store: Effect.fn("QdrantGraphEmbeddingsStore.store")(function* (message) {
|
||||
return yield* Effect.tryPromise({
|
||||
try: () => store.store(message),
|
||||
catch: (cause) => qdrantGraphEmbeddingsStoreError("store", cause),
|
||||
});
|
||||
}),
|
||||
deleteCollection: Effect.fn("QdrantGraphEmbeddingsStore.deleteCollection")(function* (
|
||||
user,
|
||||
collection,
|
||||
) {
|
||||
return yield* Effect.tryPromise({
|
||||
try: () => store.deleteCollection(user, collection),
|
||||
catch: (cause) => qdrantGraphEmbeddingsStoreError("delete-collection", cause),
|
||||
});
|
||||
}),
|
||||
};
|
||||
};
|
||||
|
||||
export const QdrantGraphEmbeddingsStoreLive = (
|
||||
config: QdrantGraphEmbeddingsConfig = {},
|
||||
): Layer.Layer<QdrantGraphEmbeddingsStoreService> =>
|
||||
Layer.succeed(
|
||||
QdrantGraphEmbeddingsStoreService,
|
||||
QdrantGraphEmbeddingsStoreService.of(makeQdrantGraphEmbeddingsStoreService(config)),
|
||||
);
|
||||
|
|
|
|||
|
|
@ -14,47 +14,72 @@ import {
|
|||
type ProcessorConfig,
|
||||
type FlowContext,
|
||||
type Triples,
|
||||
type Spec,
|
||||
} from "@trustgraph/base";
|
||||
import { makeProcessorProgram } from "@trustgraph/base";
|
||||
import { FalkorDBTriplesStore } from "./falkordb.js";
|
||||
import { makeFlowProcessorProgram } from "@trustgraph/base";
|
||||
import { Effect } from "effect";
|
||||
import {
|
||||
FalkorDBTriplesStoreLive,
|
||||
FalkorDBTriplesStoreService,
|
||||
makeFalkorDBTriplesStoreService,
|
||||
type FalkorDBConfig,
|
||||
type FalkorDBTriplesStoreError,
|
||||
} from "./falkordb.js";
|
||||
|
||||
export class TriplesStoreService extends FlowProcessor {
|
||||
private store: FalkorDBTriplesStore;
|
||||
const onStoreTriplesMessage = Effect.fn("TriplesStoreService.onMessage")(function* (
|
||||
msg: Triples,
|
||||
_properties: Record<string, string>,
|
||||
_flowCtx: FlowContext<FalkorDBTriplesStoreService>,
|
||||
): Effect.fn.Return<void, FalkorDBTriplesStoreError, FalkorDBTriplesStoreService> {
|
||||
if (msg.triples.length === 0) return;
|
||||
|
||||
const user = msg.metadata?.user ?? "default";
|
||||
const collection = msg.metadata?.collection ?? "default";
|
||||
const store = yield* FalkorDBTriplesStoreService;
|
||||
|
||||
yield* store.storeTriples(msg.triples, user, collection);
|
||||
|
||||
yield* Effect.log(
|
||||
`[TriplesStore] Stored ${msg.triples.length} triples for ${user}/${collection}`,
|
||||
);
|
||||
});
|
||||
|
||||
export const makeTriplesStoreSpecs = (): ReadonlyArray<Spec<FalkorDBTriplesStoreService>> => [
|
||||
new ConsumerSpec<Triples, FalkorDBTriplesStoreError, FalkorDBTriplesStoreService>(
|
||||
"store-triples-input",
|
||||
onStoreTriplesMessage,
|
||||
),
|
||||
];
|
||||
|
||||
export class TriplesStoreService extends FlowProcessor<FalkorDBTriplesStoreService> {
|
||||
private readonly store = makeFalkorDBTriplesStoreService();
|
||||
|
||||
constructor(config: ProcessorConfig) {
|
||||
super(config);
|
||||
this.store = new FalkorDBTriplesStore();
|
||||
|
||||
this.registerSpecification(
|
||||
ConsumerSpec.fromPromise<Triples>("store-triples-input", this.onMessage.bind(this)),
|
||||
);
|
||||
for (const spec of makeTriplesStoreSpecs()) {
|
||||
this.registerSpecification(spec);
|
||||
}
|
||||
|
||||
console.log("[TriplesStore] Service initialized");
|
||||
}
|
||||
|
||||
private async onMessage(
|
||||
msg: Triples,
|
||||
_properties: Record<string, string>,
|
||||
_flowCtx: FlowContext,
|
||||
): Promise<void> {
|
||||
if (msg.triples.length === 0) return;
|
||||
|
||||
const user = msg.metadata?.user ?? "default";
|
||||
const collection = msg.metadata?.collection ?? "default";
|
||||
|
||||
await this.store.storeTriples(msg.triples, user, collection);
|
||||
|
||||
console.log(
|
||||
`[TriplesStore] Stored ${msg.triples.length} triples for ${user}/${collection}`,
|
||||
override startEffect() {
|
||||
return super.startEffect().pipe(
|
||||
Effect.provideService(
|
||||
FalkorDBTriplesStoreService,
|
||||
FalkorDBTriplesStoreService.of(this.store),
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export const program = makeProcessorProgram({
|
||||
export const program = makeFlowProcessorProgram<ProcessorConfig & FalkorDBConfig, never, FalkorDBTriplesStoreService>({
|
||||
id: "triples-store",
|
||||
make: (config) => new TriplesStoreService(config),
|
||||
specs: () => makeTriplesStoreSpecs(),
|
||||
layer: (config) => FalkorDBTriplesStoreLive(config),
|
||||
});
|
||||
|
||||
export async function run(): Promise<void> {
|
||||
await TriplesStoreService.launch("triples-store");
|
||||
await Effect.runPromise(program);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,7 +8,9 @@
|
|||
*/
|
||||
|
||||
import { createClient, Graph } from "falkordb";
|
||||
import type { Term, Triple } from "@trustgraph/base";
|
||||
import { errorMessage, type Term, type Triple } from "@trustgraph/base";
|
||||
import { Context, Effect, Layer } from "effect";
|
||||
import * as S from "effect/Schema";
|
||||
|
||||
export interface FalkorDBConfig {
|
||||
url?: string;
|
||||
|
|
@ -130,3 +132,71 @@ export class FalkorDBTriplesStore {
|
|||
);
|
||||
}
|
||||
}
|
||||
|
||||
export class FalkorDBTriplesStoreError extends S.TaggedErrorClass<FalkorDBTriplesStoreError>()(
|
||||
"FalkorDBTriplesStoreError",
|
||||
{
|
||||
message: S.String,
|
||||
operation: S.String,
|
||||
cause: S.DefectWithStack,
|
||||
},
|
||||
) {}
|
||||
|
||||
export interface FalkorDBTriplesStoreServiceShape {
|
||||
readonly storeTriples: (
|
||||
triples: ReadonlyArray<Triple>,
|
||||
user: string,
|
||||
collection: string,
|
||||
) => Effect.Effect<void, FalkorDBTriplesStoreError>;
|
||||
readonly deleteCollection: (
|
||||
user: string,
|
||||
collection: string,
|
||||
) => Effect.Effect<void, FalkorDBTriplesStoreError>;
|
||||
}
|
||||
|
||||
export class FalkorDBTriplesStoreService extends Context.Service<
|
||||
FalkorDBTriplesStoreService,
|
||||
FalkorDBTriplesStoreServiceShape
|
||||
>()(
|
||||
"@trustgraph/flow/storage/triples/falkordb/FalkorDBTriplesStoreService",
|
||||
) {}
|
||||
|
||||
const falkorDBTriplesStoreError = (operation: string, cause: unknown) =>
|
||||
new FalkorDBTriplesStoreError({
|
||||
operation,
|
||||
message: errorMessage(cause),
|
||||
cause,
|
||||
});
|
||||
|
||||
export const makeFalkorDBTriplesStoreService = (
|
||||
config: FalkorDBConfig = {},
|
||||
): FalkorDBTriplesStoreServiceShape => {
|
||||
const store = new FalkorDBTriplesStore(config);
|
||||
return {
|
||||
storeTriples: Effect.fn("FalkorDBTriplesStore.storeTriples")((
|
||||
triples: ReadonlyArray<Triple>,
|
||||
user: string,
|
||||
collection: string,
|
||||
) =>
|
||||
Effect.tryPromise({
|
||||
try: () => store.storeTriples(Array.from(triples), user, collection),
|
||||
catch: (cause) => falkorDBTriplesStoreError("store-triples", cause),
|
||||
})),
|
||||
deleteCollection: Effect.fn("FalkorDBTriplesStore.deleteCollection")((
|
||||
user: string,
|
||||
collection: string,
|
||||
) =>
|
||||
Effect.tryPromise({
|
||||
try: () => store.deleteCollection(user, collection),
|
||||
catch: (cause) => falkorDBTriplesStoreError("delete-collection", cause),
|
||||
})),
|
||||
};
|
||||
};
|
||||
|
||||
export const FalkorDBTriplesStoreLive = (
|
||||
config: FalkorDBConfig = {},
|
||||
): Layer.Layer<FalkorDBTriplesStoreService> =>
|
||||
Layer.succeed(
|
||||
FalkorDBTriplesStoreService,
|
||||
FalkorDBTriplesStoreService.of(makeFalkorDBTriplesStoreService(config)),
|
||||
);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue