mirror of
https://github.com/trustgraph-ai/trustgraph.git
synced 2026-07-02 02:58:10 +02:00
init
This commit is contained in:
elpresidank
parent
c386f68743
commit
b6536eca38
100 changed files with 17680 additions and 377 deletions
106
ts/packages/flow/src/storage/embeddings/qdrant-doc.ts
Normal file
106
ts/packages/flow/src/storage/embeddings/qdrant-doc.ts
Normal file
|
|
@ -0,0 +1,106 @@
|
|||
/**
|
||||
* Qdrant document embeddings write service.
|
||||
*
|
||||
* Stores document chunk embeddings in Qdrant for later similarity search.
|
||||
* Collection naming: d_{user}_{collection}_{dimension}
|
||||
* Collections are lazily created on first write with cosine distance.
|
||||
*
|
||||
* Python reference: trustgraph-flow/trustgraph/storage/doc_embeddings/qdrant/write.py
|
||||
*/
|
||||
|
||||
import { QdrantClient } from "@qdrant/js-client-rest";
|
||||
import { randomUUID } from "node:crypto";
|
||||
|
||||
export interface QdrantDocEmbeddingsConfig {
|
||||
url?: string;
|
||||
apiKey?: string;
|
||||
}
|
||||
|
||||
export interface DocEmbeddingChunk {
|
||||
chunkId: string;
|
||||
vector: number[];
|
||||
}
|
||||
|
||||
export interface DocEmbeddingsMessage {
|
||||
user: string;
|
||||
collection: string;
|
||||
chunks: DocEmbeddingChunk[];
|
||||
}
|
||||
|
||||
export class QdrantDocEmbeddingsStore {
|
||||
private client: QdrantClient;
|
||||
private knownCollections = new Set<string>();
|
||||
|
||||
constructor(config: QdrantDocEmbeddingsConfig = {}) {
|
||||
const url = config.url ?? process.env.QDRANT_URL ?? "http://localhost:6333";
|
||||
const apiKey = config.apiKey ?? process.env.QDRANT_API_KEY;
|
||||
|
||||
this.client = new QdrantClient({ url, apiKey });
|
||||
|
||||
console.log("[QdrantDocEmbeddings] Store initialized");
|
||||
}
|
||||
|
||||
private collectionName(user: string, collection: string, dim: number): string {
|
||||
return `d_${user}_${collection}_${dim}`;
|
||||
}
|
||||
|
||||
private async ensureCollection(name: string, dim: number): Promise<void> {
|
||||
if (this.knownCollections.has(name)) return;
|
||||
|
||||
const exists = await this.client.collectionExists(name);
|
||||
if (!exists.exists) {
|
||||
console.log(`[QdrantDocEmbeddings] Creating collection ${name} (dim=${dim})`);
|
||||
await this.client.createCollection(name, {
|
||||
vectors: { size: dim, distance: "Cosine" },
|
||||
});
|
||||
}
|
||||
|
||||
this.knownCollections.add(name);
|
||||
}
|
||||
|
||||
async store(message: DocEmbeddingsMessage): Promise<void> {
|
||||
for (const chunk of message.chunks) {
|
||||
if (!chunk.chunkId || chunk.chunkId === "") continue;
|
||||
if (!chunk.vector || chunk.vector.length === 0) continue;
|
||||
|
||||
const dim = chunk.vector.length;
|
||||
const name = this.collectionName(message.user, message.collection, dim);
|
||||
|
||||
await this.ensureCollection(name, dim);
|
||||
|
||||
await this.client.upsert(name, {
|
||||
points: [
|
||||
{
|
||||
id: randomUUID(),
|
||||
vector: chunk.vector,
|
||||
payload: { chunk_id: chunk.chunkId },
|
||||
},
|
||||
],
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async deleteCollection(user: string, collection: string): Promise<void> {
|
||||
const prefix = `d_${user}_${collection}_`;
|
||||
|
||||
const allCollections = await this.client.getCollections();
|
||||
const matching = allCollections.collections.filter((c) =>
|
||||
c.name.startsWith(prefix),
|
||||
);
|
||||
|
||||
if (matching.length === 0) {
|
||||
console.log(`[QdrantDocEmbeddings] No collections matching prefix ${prefix}`);
|
||||
return;
|
||||
}
|
||||
|
||||
for (const coll of matching) {
|
||||
await this.client.deleteCollection(coll.name);
|
||||
this.knownCollections.delete(coll.name);
|
||||
console.log(`[QdrantDocEmbeddings] Deleted collection: ${coll.name}`);
|
||||
}
|
||||
|
||||
console.log(
|
||||
`[QdrantDocEmbeddings] Deleted ${matching.length} collection(s) for ${user}/${collection}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
127
ts/packages/flow/src/storage/embeddings/qdrant-graph.ts
Normal file
127
ts/packages/flow/src/storage/embeddings/qdrant-graph.ts
Normal file
|
|
@ -0,0 +1,127 @@
|
|||
/**
|
||||
* Qdrant graph embeddings write service.
|
||||
*
|
||||
* Stores entity/vector pairs in Qdrant for graph embeddings lookup.
|
||||
* Collection naming: t_{user}_{collection}_{dimension}
|
||||
* Collections are lazily created on first write with cosine distance.
|
||||
*
|
||||
* Python reference: trustgraph-flow/trustgraph/storage/graph_embeddings/qdrant/write.py
|
||||
*/
|
||||
|
||||
import { QdrantClient } from "@qdrant/js-client-rest";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import type { Term } from "@trustgraph/base";
|
||||
|
||||
export interface QdrantGraphEmbeddingsConfig {
|
||||
url?: string;
|
||||
apiKey?: string;
|
||||
}
|
||||
|
||||
export interface GraphEmbeddingEntity {
|
||||
entity: Term;
|
||||
vector: number[];
|
||||
chunkId?: string;
|
||||
}
|
||||
|
||||
export interface GraphEmbeddingsMessage {
|
||||
user: string;
|
||||
collection: string;
|
||||
entities: GraphEmbeddingEntity[];
|
||||
}
|
||||
|
||||
function getTermValue(term: Term): string | null {
|
||||
switch (term.type) {
|
||||
case "IRI":
|
||||
return term.iri;
|
||||
case "LITERAL":
|
||||
return term.value;
|
||||
case "BLANK":
|
||||
return term.id;
|
||||
case "TRIPLE":
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export class QdrantGraphEmbeddingsStore {
|
||||
private client: QdrantClient;
|
||||
private knownCollections = new Set<string>();
|
||||
|
||||
constructor(config: QdrantGraphEmbeddingsConfig = {}) {
|
||||
const url = config.url ?? process.env.QDRANT_URL ?? "http://localhost:6333";
|
||||
const apiKey = config.apiKey ?? process.env.QDRANT_API_KEY;
|
||||
|
||||
this.client = new QdrantClient({ url, apiKey });
|
||||
|
||||
console.log("[QdrantGraphEmbeddings] Store initialized");
|
||||
}
|
||||
|
||||
private collectionName(user: string, collection: string, dim: number): string {
|
||||
return `t_${user}_${collection}_${dim}`;
|
||||
}
|
||||
|
||||
private async ensureCollection(name: string, dim: number): Promise<void> {
|
||||
if (this.knownCollections.has(name)) return;
|
||||
|
||||
const exists = await this.client.collectionExists(name);
|
||||
if (!exists.exists) {
|
||||
console.log(`[QdrantGraphEmbeddings] Creating collection ${name} (dim=${dim})`);
|
||||
await this.client.createCollection(name, {
|
||||
vectors: { size: dim, distance: "Cosine" },
|
||||
});
|
||||
}
|
||||
|
||||
this.knownCollections.add(name);
|
||||
}
|
||||
|
||||
async store(message: GraphEmbeddingsMessage): Promise<void> {
|
||||
for (const entry of message.entities) {
|
||||
const entityValue = getTermValue(entry.entity);
|
||||
if (!entityValue || entityValue === "") continue;
|
||||
if (!entry.vector || entry.vector.length === 0) continue;
|
||||
|
||||
const dim = entry.vector.length;
|
||||
const name = this.collectionName(message.user, message.collection, dim);
|
||||
|
||||
await this.ensureCollection(name, dim);
|
||||
|
||||
const payload: Record<string, unknown> = { entity: entityValue };
|
||||
if (entry.chunkId) {
|
||||
payload.chunk_id = entry.chunkId;
|
||||
}
|
||||
|
||||
await this.client.upsert(name, {
|
||||
points: [
|
||||
{
|
||||
id: randomUUID(),
|
||||
vector: entry.vector,
|
||||
payload,
|
||||
},
|
||||
],
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async deleteCollection(user: string, collection: string): Promise<void> {
|
||||
const prefix = `t_${user}_${collection}_`;
|
||||
|
||||
const allCollections = await this.client.getCollections();
|
||||
const matching = allCollections.collections.filter((c) =>
|
||||
c.name.startsWith(prefix),
|
||||
);
|
||||
|
||||
if (matching.length === 0) {
|
||||
console.log(`[QdrantGraphEmbeddings] No collections matching prefix ${prefix}`);
|
||||
return;
|
||||
}
|
||||
|
||||
for (const coll of matching) {
|
||||
await this.client.deleteCollection(coll.name);
|
||||
this.knownCollections.delete(coll.name);
|
||||
console.log(`[QdrantGraphEmbeddings] Deleted collection: ${coll.name}`);
|
||||
}
|
||||
|
||||
console.log(
|
||||
`[QdrantGraphEmbeddings] Deleted ${matching.length} collection(s) for ${user}/${collection}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
116
ts/packages/flow/src/storage/triples/falkordb.ts
Normal file
116
ts/packages/flow/src/storage/triples/falkordb.ts
Normal file
|
|
@ -0,0 +1,116 @@
|
|||
/**
|
||||
* FalkorDB triples store — writes RDF triples to a FalkorDB graph.
|
||||
*
|
||||
* FalkorDB is Redis-based and uses Cypher queries, same as the Python impl.
|
||||
* Pairs well with Graphiti which also uses FalkorDB as its backend.
|
||||
*
|
||||
* Python reference: trustgraph-flow/trustgraph/storage/triples/falkordb/write.py
|
||||
*/
|
||||
|
||||
import { createClient, Graph } from "falkordb";
|
||||
import type { Term, Triple } from "@trustgraph/base";
|
||||
|
||||
export interface FalkorDBConfig {
|
||||
url?: string;
|
||||
database?: string;
|
||||
}
|
||||
|
||||
function getTermValue(term: Term): string {
|
||||
switch (term.type) {
|
||||
case "IRI":
|
||||
return term.iri;
|
||||
case "LITERAL":
|
||||
return term.value;
|
||||
case "BLANK":
|
||||
return term.id;
|
||||
case "TRIPLE":
|
||||
return getTermValue(term.triple.s); // fallback
|
||||
}
|
||||
}
|
||||
|
||||
export class FalkorDBTriplesStore {
|
||||
private graph: Graph;
|
||||
|
||||
constructor(config: FalkorDBConfig = {}) {
|
||||
const url = config.url ?? process.env.FALKORDB_URL ?? "redis://localhost:6379";
|
||||
const database = config.database ?? "falkordb";
|
||||
|
||||
const client = createClient({ url });
|
||||
this.graph = new Graph(client, database);
|
||||
}
|
||||
|
||||
async createNode(uri: string, user: string, collection: string): Promise<void> {
|
||||
await this.graph.query(
|
||||
"MERGE (n:Node {uri: $uri, user: $user, collection: $collection})",
|
||||
{ params: { uri, user, collection } },
|
||||
);
|
||||
}
|
||||
|
||||
async createLiteral(value: string, user: string, collection: string): Promise<void> {
|
||||
await this.graph.query(
|
||||
"MERGE (n:Literal {value: $value, user: $user, collection: $collection})",
|
||||
{ params: { value, user, collection } },
|
||||
);
|
||||
}
|
||||
|
||||
async relateNode(
|
||||
src: string, uri: string, dest: string,
|
||||
user: string, collection: string,
|
||||
): Promise<void> {
|
||||
await this.graph.query(
|
||||
"MATCH (src:Node {uri: $src, user: $user, collection: $collection}) " +
|
||||
"MATCH (dest:Node {uri: $dest, user: $user, collection: $collection}) " +
|
||||
"MERGE (src)-[:Rel {uri: $uri, user: $user, collection: $collection}]->(dest)",
|
||||
{ params: { src, dest, uri, user, collection } },
|
||||
);
|
||||
}
|
||||
|
||||
async relateLiteral(
|
||||
src: string, uri: string, dest: string,
|
||||
user: string, collection: string,
|
||||
): Promise<void> {
|
||||
await this.graph.query(
|
||||
"MATCH (src:Node {uri: $src, user: $user, collection: $collection}) " +
|
||||
"MATCH (dest:Literal {value: $dest, user: $user, collection: $collection}) " +
|
||||
"MERGE (src)-[:Rel {uri: $uri, user: $user, collection: $collection}]->(dest)",
|
||||
{ params: { src, dest, uri, user, collection } },
|
||||
);
|
||||
}
|
||||
|
||||
async storeTriples(
|
||||
triples: Triple[],
|
||||
user = "default",
|
||||
collection = "default",
|
||||
): Promise<void> {
|
||||
for (const t of triples) {
|
||||
const s = getTermValue(t.s);
|
||||
const p = getTermValue(t.p);
|
||||
const o = getTermValue(t.o);
|
||||
|
||||
await this.createNode(s, user, collection);
|
||||
|
||||
if (t.o.type === "IRI") {
|
||||
await this.createNode(o, user, collection);
|
||||
await this.relateNode(s, p, o, user, collection);
|
||||
} else {
|
||||
await this.createLiteral(o, user, collection);
|
||||
await this.relateLiteral(s, p, o, user, collection);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async deleteCollection(user: string, collection: string): Promise<void> {
|
||||
await this.graph.query(
|
||||
"MATCH (n:Node {user: $user, collection: $collection}) DETACH DELETE n",
|
||||
{ params: { user, collection } },
|
||||
);
|
||||
await this.graph.query(
|
||||
"MATCH (n:Literal {user: $user, collection: $collection}) DETACH DELETE n",
|
||||
{ params: { user, collection } },
|
||||
);
|
||||
await this.graph.query(
|
||||
"MATCH (c:CollectionMetadata {user: $user, collection: $collection}) DELETE c",
|
||||
{ params: { user, collection } },
|
||||
);
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue