apunkt/trustgraph
1
0
Fork 0
mirror of https://github.com/trustgraph-ai/trustgraph.git synced 2026-07-02 22:41:01 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 5

Remove native classes from TS runtime

Browse source
This commit is contained in:
elpresidank 2026-06-01 20:26:47 -05:00
parent 952daf325d
commit dca2786828
79 changed files with 7622 additions and 6703 deletions
Download patch file Download diff file Expand all files Collapse all files

46
ts/packages/flow/src/storage/embeddings/graph-embeddings-service.ts
View file

@ -10,10 +10,11 @@
*/
import {
FlowProcessor,
ConsumerSpec,
RequestResponseSpec,
makeFlowProcessor,
makeConsumerSpec,
makeRequestResponseSpec,
type ProcessorConfig,
type FlowProcessorRuntime,
type FlowContext,
type FlowResourceNotFoundError,
type MessagingDeliveryError,
@ -77,40 +78,37 @@ const onGraphEmbeddingsStoreMessage = Effect.fn("GraphEmbeddingsStoreService.onM
});
export const makeGraphEmbeddingsStoreSpecs = (): ReadonlyArray<Spec<GraphEmbeddingsStoreRequirements>> => [
new ConsumerSpec<EntityContexts, GraphEmbeddingsStoreError, GraphEmbeddingsStoreRequirements>(
makeConsumerSpec<EntityContexts, GraphEmbeddingsStoreError, GraphEmbeddingsStoreRequirements>(
"store-graph-embeddings-input",
onGraphEmbeddingsStoreMessage,
),
new RequestResponseSpec<EmbeddingsRequest, EmbeddingsResponse>(
makeRequestResponseSpec<EmbeddingsRequest, EmbeddingsResponse>(
"embeddings-client",
"embeddings-request",
"embeddings-response",
),
];
export class GraphEmbeddingsStoreService extends FlowProcessor<GraphEmbeddingsStoreRequirements> {
private readonly store = makeQdrantGraphEmbeddingsStoreService();
export type GraphEmbeddingsStoreService = FlowProcessorRuntime<GraphEmbeddingsStoreRequirements>;
constructor(config: ProcessorConfig) {
super(config);
for (const spec of makeGraphEmbeddingsStoreSpecs()) {
this.registerSpecification(spec);
}
console.log("[GraphEmbeddingsStore] Service initialized");
}
override startEffect() {
return super.startEffect().pipe(
Effect.provideService(
QdrantGraphEmbeddingsStoreService,
QdrantGraphEmbeddingsStoreService.of(this.store),
export function makeGraphEmbeddingsStoreService(config: ProcessorConfig): GraphEmbeddingsStoreService {
const store = makeQdrantGraphEmbeddingsStoreService();
const service = makeFlowProcessor(config, {
specifications: makeGraphEmbeddingsStoreSpecs(),
provide: (effect) =>
effect.pipe(
Effect.provideService(
QdrantGraphEmbeddingsStoreService,
QdrantGraphEmbeddingsStoreService.of(store),
),
),
);
}
});
console.log("[GraphEmbeddingsStore] Service initialized");
return service;
}
export const GraphEmbeddingsStoreService = makeGraphEmbeddingsStoreService;
export const program = makeFlowProcessorProgram<
ProcessorConfig & QdrantGraphEmbeddingsConfig,
never,

66
ts/packages/flow/src/storage/embeddings/qdrant-doc.ts
View file

@ -27,51 +27,53 @@ export interface DocEmbeddingsMessage {
chunks: DocEmbeddingChunk[];
}
export class QdrantDocEmbeddingsStore {
private client: QdrantClient;
private knownCollections = new Set<string>();
export interface QdrantDocEmbeddingsStore {
readonly store: (message: DocEmbeddingsMessage) => Promise<void>;
readonly deleteCollection: (user: string, collection: string) => Promise<void>;
}
constructor(config: QdrantDocEmbeddingsConfig = {}) {
const url = config.url ?? process.env.QDRANT_URL ?? "http://localhost:6333";
const apiKey = config.apiKey ?? process.env.QDRANT_API_KEY;
export function makeQdrantDocEmbeddingsStore(
config: QdrantDocEmbeddingsConfig = {},
): QdrantDocEmbeddingsStore {
const url = config.url ?? process.env.QDRANT_URL ?? "http://localhost:6333";
const apiKey = config.apiKey ?? process.env.QDRANT_API_KEY;
this.client = new QdrantClient({
url,
...(apiKey !== undefined && apiKey.length > 0 ? { apiKey } : {}),
});
const client = new QdrantClient({
url,
...(apiKey !== undefined && apiKey.length > 0 ? { apiKey } : {}),
});
const knownCollections = new Set<string>();
console.log("[QdrantDocEmbeddings] Store initialized");
}
console.log("[QdrantDocEmbeddings] Store initialized");
private collectionName(user: string, collection: string, dim: number): string {
return `d_${user}_${collection}_${dim}`;
}
const collectionName = (user: string, collection: string, dim: number): string =>
`d_${user}_${collection}_${dim}`;
private async ensureCollection(name: string, dim: number): Promise<void> {
if (this.knownCollections.has(name)) return;
const ensureCollection = async (name: string, dim: number): Promise<void> => {
if (knownCollections.has(name)) return;
const exists = await this.client.collectionExists(name);
const exists = await client.collectionExists(name);
if (!exists.exists) {
console.log(`[QdrantDocEmbeddings] Creating collection ${name} (dim=${dim})`);
await this.client.createCollection(name, {
await client.createCollection(name, {
vectors: { size: dim, distance: "Cosine" },
});
}
this.knownCollections.add(name);
}
knownCollections.add(name);
};
async store(message: DocEmbeddingsMessage): Promise<void> {
const store = async (message: DocEmbeddingsMessage): Promise<void> => {
for (const chunk of message.chunks) {
if (chunk.chunkId.length === 0) continue;
if (chunk.vector.length === 0) continue;
const dim = chunk.vector.length;
const name = this.collectionName(message.user, message.collection, dim);
const name = collectionName(message.user, message.collection, dim);
await this.ensureCollection(name, dim);
await ensureCollection(name, dim);
await this.client.upsert(name, {
await client.upsert(name, {
points: [
{
id: crypto.randomUUID(),
@ -86,12 +88,12 @@ export class QdrantDocEmbeddingsStore {
],
});
}
}
};
async deleteCollection(user: string, collection: string): Promise<void> {
const deleteCollection = async (user: string, collection: string): Promise<void> => {
const prefix = `d_${user}_${collection}_`;
const allCollections = await this.client.getCollections();
const allCollections = await client.getCollections();
const matching = allCollections.collections.filter((c) =>
c.name.startsWith(prefix),
);
@ -102,13 +104,15 @@ export class QdrantDocEmbeddingsStore {
}
for (const coll of matching) {
await this.client.deleteCollection(coll.name);
this.knownCollections.delete(coll.name);
await client.deleteCollection(coll.name);
knownCollections.delete(coll.name);
console.log(`[QdrantDocEmbeddings] Deleted collection: ${coll.name}`);
}
console.log(
`[QdrantDocEmbeddings] Deleted ${matching.length} collection(s) for ${user}/${collection}`,
);
}
};
return { store, deleteCollection };
}

68
ts/packages/flow/src/storage/embeddings/qdrant-graph.ts
View file

@ -43,57 +43,59 @@ function getTermValue(term: Term): string | null {
}
}
export class QdrantGraphEmbeddingsStore {
private client: QdrantClient;
private knownCollections = new Set<string>();
export interface QdrantGraphEmbeddingsStore {
readonly store: (message: GraphEmbeddingsMessage) => Promise<void>;
readonly deleteCollection: (user: string, collection: string) => Promise<void>;
}
constructor(config: QdrantGraphEmbeddingsConfig = {}) {
const url = config.url ?? process.env.QDRANT_URL ?? "http://localhost:6333";
const apiKey = config.apiKey ?? process.env.QDRANT_API_KEY;
export function makeQdrantGraphEmbeddingsStore(
config: QdrantGraphEmbeddingsConfig = {},
): QdrantGraphEmbeddingsStore {
const url = config.url ?? process.env.QDRANT_URL ?? "http://localhost:6333";
const apiKey = config.apiKey ?? process.env.QDRANT_API_KEY;
this.client = new QdrantClient({
url,
...(apiKey !== undefined && apiKey.length > 0 ? { apiKey } : {}),
});
const client = new QdrantClient({
url,
...(apiKey !== undefined && apiKey.length > 0 ? { apiKey } : {}),
});
const knownCollections = new Set<string>();
console.log("[QdrantGraphEmbeddings] Store initialized");
}
console.log("[QdrantGraphEmbeddings] Store initialized");
private collectionName(user: string, collection: string, dim: number): string {
return `t_${user}_${collection}_${dim}`;
}
const collectionName = (user: string, collection: string, dim: number): string =>
`t_${user}_${collection}_${dim}`;
private async ensureCollection(name: string, dim: number): Promise<void> {
if (this.knownCollections.has(name)) return;
const ensureCollection = async (name: string, dim: number): Promise<void> => {
if (knownCollections.has(name)) return;
const exists = await this.client.collectionExists(name);
const exists = await client.collectionExists(name);
if (!exists.exists) {
console.log(`[QdrantGraphEmbeddings] Creating collection ${name} (dim=${dim})`);
await this.client.createCollection(name, {
await client.createCollection(name, {
vectors: { size: dim, distance: "Cosine" },
});
}
this.knownCollections.add(name);
}
knownCollections.add(name);
};
async store(message: GraphEmbeddingsMessage): Promise<void> {
const store = async (message: GraphEmbeddingsMessage): Promise<void> => {
for (const entry of message.entities) {
const entityValue = getTermValue(entry.entity);
if (entityValue === null || entityValue.length === 0) continue;
if (entry.vector.length === 0) continue;
const dim = entry.vector.length;
const name = this.collectionName(message.user, message.collection, dim);
const name = collectionName(message.user, message.collection, dim);
await this.ensureCollection(name, dim);
await ensureCollection(name, dim);
const payload: Record<string, unknown> = { entity: entityValue };
if (entry.chunkId !== undefined && entry.chunkId.length > 0) {
payload.chunk_id = entry.chunkId;
}
await this.client.upsert(name, {
await client.upsert(name, {
points: [
{
id: crypto.randomUUID(),
@ -103,12 +105,12 @@ export class QdrantGraphEmbeddingsStore {
],
});
}
}
};
async deleteCollection(user: string, collection: string): Promise<void> {
const deleteCollection = async (user: string, collection: string): Promise<void> => {
const prefix = `t_${user}_${collection}_`;
const allCollections = await this.client.getCollections();
const allCollections = await client.getCollections();
const matching = allCollections.collections.filter((c) =>
c.name.startsWith(prefix),
);
@ -119,15 +121,17 @@ export class QdrantGraphEmbeddingsStore {
}
for (const coll of matching) {
await this.client.deleteCollection(coll.name);
this.knownCollections.delete(coll.name);
await client.deleteCollection(coll.name);
knownCollections.delete(coll.name);
console.log(`[QdrantGraphEmbeddings] Deleted collection: ${coll.name}`);
}
console.log(
`[QdrantGraphEmbeddings] Deleted ${matching.length} collection(s) for ${user}/${collection}`,
);
}
};
return { store, deleteCollection };
}
export class QdrantGraphEmbeddingsStoreError extends S.TaggedErrorClass<QdrantGraphEmbeddingsStoreError>()(
@ -166,7 +170,7 @@ const qdrantGraphEmbeddingsStoreError = (operation: string, cause: unknown) =>
export const makeQdrantGraphEmbeddingsStoreService = (
config: QdrantGraphEmbeddingsConfig = {},
): QdrantGraphEmbeddingsStoreServiceShape => {
const store = new QdrantGraphEmbeddingsStore(config);
const store = makeQdrantGraphEmbeddingsStore(config);
return {
store: Effect.fn("QdrantGraphEmbeddingsStore.store")(function* (message) {
return yield* Effect.tryPromise({

42
ts/packages/flow/src/storage/triples/falkordb-service.ts
View file

@ -9,9 +9,10 @@
*/
import {
FlowProcessor,
ConsumerSpec,
makeFlowProcessor,
makeConsumerSpec,
type ProcessorConfig,
type FlowProcessorRuntime,
type FlowContext,
type Triples,
type Spec,
@ -45,35 +46,32 @@ const onStoreTriplesMessage = Effect.fn("TriplesStoreService.onMessage")(functio
});
export const makeTriplesStoreSpecs = (): ReadonlyArray<Spec<FalkorDBTriplesStoreService>> => [
new ConsumerSpec<Triples, FalkorDBTriplesStoreError, FalkorDBTriplesStoreService>(
makeConsumerSpec<Triples, FalkorDBTriplesStoreError, FalkorDBTriplesStoreService>(
"store-triples-input",
onStoreTriplesMessage,
),
];
export class TriplesStoreService extends FlowProcessor<FalkorDBTriplesStoreService> {
private readonly store = makeFalkorDBTriplesStoreService();
export type TriplesStoreService = FlowProcessorRuntime<FalkorDBTriplesStoreService>;
constructor(config: ProcessorConfig) {
super(config);
for (const spec of makeTriplesStoreSpecs()) {
this.registerSpecification(spec);
}
console.log("[TriplesStore] Service initialized");
}
override startEffect() {
return super.startEffect().pipe(
Effect.provideService(
FalkorDBTriplesStoreService,
FalkorDBTriplesStoreService.of(this.store),
export function makeTriplesStoreService(config: ProcessorConfig): TriplesStoreService {
const store = makeFalkorDBTriplesStoreService();
const service = makeFlowProcessor(config, {
specifications: makeTriplesStoreSpecs(),
provide: (effect) =>
effect.pipe(
Effect.provideService(
FalkorDBTriplesStoreService,
FalkorDBTriplesStoreService.of(store),
),
),
);
}
});
console.log("[TriplesStore] Service initialized");
return service;
}
export const TriplesStoreService = makeTriplesStoreService;
export const program = makeFlowProcessorProgram<ProcessorConfig & FalkorDBConfig, never, FalkorDBTriplesStoreService>({
id: "triples-store",
specs: () => makeTriplesStoreSpecs(),

131
ts/packages/flow/src/storage/triples/falkordb.ts
View file

@ -30,107 +30,136 @@ function getTermValue(term: Term): string {
}
}
export class FalkorDBTriplesStore {
private graph: Graph;
private connectPromise: Promise<void>;
export interface FalkorDBTriplesStore {
readonly createNode: (uri: string, user: string, collection: string) => Promise<void>;
readonly createLiteral: (value: string, user: string, collection: string) => Promise<void>;
readonly relateNode: (
src: string,
uri: string,
dest: string,
user: string,
collection: string,
) => Promise<void>;
readonly relateLiteral: (
src: string,
uri: string,
dest: string,
user: string,
collection: string,
) => Promise<void>;
readonly storeTriples: (
triples: Triple[],
user?: string,
collection?: string,
) => Promise<void>;
readonly deleteCollection: (user: string, collection: string) => Promise<void>;
}
constructor(config: FalkorDBConfig = {}) {
const url = config.url ?? process.env.FALKORDB_URL ?? "redis://localhost:6379";
const database = config.database ?? "falkordb";
export function makeFalkorDBTriplesStore(config: FalkorDBConfig = {}): FalkorDBTriplesStore {
const url = config.url ?? process.env.FALKORDB_URL ?? "redis://localhost:6379";
const database = config.database ?? "falkordb";
const client = createClient({ url });
this.graph = new Graph(client, database);
this.connectPromise = client.connect().then(() => {
console.log(`[FalkorDBTriplesStore] Connected to ${url}, graph: ${database}`);
}).catch((err) => {
console.error(`[FalkorDBTriplesStore] Connection failed:`, err);
throw err;
});
}
const client = createClient({ url });
const graph = new Graph(client, database);
const connectPromise = client.connect().then(() => {
console.log(`[FalkorDBTriplesStore] Connected to ${url}, graph: ${database}`);
}).catch((err) => {
console.error(`[FalkorDBTriplesStore] Connection failed:`, err);
throw err;
});
private async ensureConnected(): Promise<void> {
await this.connectPromise;
}
const ensureConnected = async (): Promise<void> => {
await connectPromise;
};
async createNode(uri: string, user: string, collection: string): Promise<void> {
await this.ensureConnected();
await this.graph.query(
const createNode = async (uri: string, user: string, collection: string): Promise<void> => {
await ensureConnected();
await graph.query(
"MERGE (n:Node {uri: $uri, user: $user, collection: $collection})",
{ params: { uri, user, collection } },
);
}
};
async createLiteral(value: string, user: string, collection: string): Promise<void> {
await this.ensureConnected();
await this.graph.query(
const createLiteral = async (value: string, user: string, collection: string): Promise<void> => {
await ensureConnected();
await graph.query(
"MERGE (n:Literal {value: $value, user: $user, collection: $collection})",
{ params: { value, user, collection } },
);
}
};
async relateNode(
const relateNode = async (
src: string, uri: string, dest: string,
user: string, collection: string,
): Promise<void> {
await this.ensureConnected();
await this.graph.query(
): Promise<void> => {
await ensureConnected();
await graph.query(
"MATCH (src:Node {uri: $src, user: $user, collection: $collection}) " +
"MATCH (dest:Node {uri: $dest, user: $user, collection: $collection}) " +
"MERGE (src)-[:Rel {uri: $uri, user: $user, collection: $collection}]->(dest)",
{ params: { src, dest, uri, user, collection } },
);
}
};
async relateLiteral(
const relateLiteral = async (
src: string, uri: string, dest: string,
user: string, collection: string,
): Promise<void> {
await this.ensureConnected();
await this.graph.query(
): Promise<void> => {
await ensureConnected();
await graph.query(
"MATCH (src:Node {uri: $src, user: $user, collection: $collection}) " +
"MATCH (dest:Literal {value: $dest, user: $user, collection: $collection}) " +
"MERGE (src)-[:Rel {uri: $uri, user: $user, collection: $collection}]->(dest)",
{ params: { src, dest, uri, user, collection } },
);
}
};
async storeTriples(
const storeTriples = async (
triples: Triple[],
user = "default",
collection = "default",
): Promise<void> {
): Promise<void> => {
for (const t of triples) {
const s = getTermValue(t.s);
const p = getTermValue(t.p);
const o = getTermValue(t.o);
await this.createNode(s, user, collection);
await createNode(s, user, collection);
if (t.o.type === "IRI") {
await this.createNode(o, user, collection);
await this.relateNode(s, p, o, user, collection);
await createNode(o, user, collection);
await relateNode(s, p, o, user, collection);
} else {
await this.createLiteral(o, user, collection);
await this.relateLiteral(s, p, o, user, collection);
await createLiteral(o, user, collection);
await relateLiteral(s, p, o, user, collection);
}
}
}
};
async deleteCollection(user: string, collection: string): Promise<void> {
await this.ensureConnected();
await this.graph.query(
const deleteCollection = async (user: string, collection: string): Promise<void> => {
await ensureConnected();
await graph.query(
"MATCH (n:Node {user: $user, collection: $collection}) DETACH DELETE n",
{ params: { user, collection } },
);
await this.graph.query(
await graph.query(
"MATCH (n:Literal {user: $user, collection: $collection}) DETACH DELETE n",
{ params: { user, collection } },
);
await this.graph.query(
await graph.query(
"MATCH (c:CollectionMetadata {user: $user, collection: $collection}) DELETE c",
{ params: { user, collection } },
);
}
};
return {
createNode,
createLiteral,
relateNode,
relateLiteral,
storeTriples,
deleteCollection,
};
}
export class FalkorDBTriplesStoreError extends S.TaggedErrorClass<FalkorDBTriplesStoreError>()(
@ -171,7 +200,7 @@ const falkorDBTriplesStoreError = (operation: string, cause: unknown) =>
export const makeFalkorDBTriplesStoreService = (
config: FalkorDBConfig = {},
): FalkorDBTriplesStoreServiceShape => {
const store = new FalkorDBTriplesStore(config);
const store = makeFalkorDBTriplesStore(config);
return {
storeTriples: Effect.fn("FalkorDBTriplesStore.storeTriples")((
triples: ReadonlyArray<Triple>,