init

ts/packages/flow/src/storage/embeddings/qdrant-doc.ts

@@ -0,0 +1,106 @@
+/**
+ * Qdrant document embeddings write service.
+ *
+ * Stores document chunk embeddings in Qdrant for later similarity search.
+ * Collection naming: d_{user}_{collection}_{dimension}
+ * Collections are lazily created on first write with cosine distance.
+ *
+ * Python reference: trustgraph-flow/trustgraph/storage/doc_embeddings/qdrant/write.py
+ */
+
+import { QdrantClient } from "@qdrant/js-client-rest";
+import { randomUUID } from "node:crypto";
+
+export interface QdrantDocEmbeddingsConfig {
+  url?: string;
+  apiKey?: string;
+}
+
+export interface DocEmbeddingChunk {
+  chunkId: string;
+  vector: number[];
+}
+
+export interface DocEmbeddingsMessage {
+  user: string;
+  collection: string;
+  chunks: DocEmbeddingChunk[];
+}
+
+export class QdrantDocEmbeddingsStore {
+  private client: QdrantClient;
+  private knownCollections = new Set<string>();
+
+  constructor(config: QdrantDocEmbeddingsConfig = {}) {
+    const url = config.url ?? process.env.QDRANT_URL ?? "http://localhost:6333";
+    const apiKey = config.apiKey ?? process.env.QDRANT_API_KEY;
+
+    this.client = new QdrantClient({ url, apiKey });
+
+    console.log("[QdrantDocEmbeddings] Store initialized");
+  }
+
+  private collectionName(user: string, collection: string, dim: number): string {
+    return `d_${user}_${collection}_${dim}`;
+  }
+
+  private async ensureCollection(name: string, dim: number): Promise<void> {
+    if (this.knownCollections.has(name)) return;
+
+    const exists = await this.client.collectionExists(name);
+    if (!exists.exists) {
+      console.log(`[QdrantDocEmbeddings] Creating collection ${name} (dim=${dim})`);
+      await this.client.createCollection(name, {
+        vectors: { size: dim, distance: "Cosine" },
+      });
+    }
+
+    this.knownCollections.add(name);
+  }
+
+  async store(message: DocEmbeddingsMessage): Promise<void> {
+    for (const chunk of message.chunks) {
+      if (!chunk.chunkId || chunk.chunkId === "") continue;
+      if (!chunk.vector || chunk.vector.length === 0) continue;
+
+      const dim = chunk.vector.length;
+      const name = this.collectionName(message.user, message.collection, dim);
+
+      await this.ensureCollection(name, dim);
+
+      await this.client.upsert(name, {
+        points: [
+          {
+            id: randomUUID(),
+            vector: chunk.vector,
+            payload: { chunk_id: chunk.chunkId },
+          },
+        ],
+      });
+    }
+  }
+
+  async deleteCollection(user: string, collection: string): Promise<void> {
+    const prefix = `d_${user}_${collection}_`;
+
+    const allCollections = await this.client.getCollections();
+    const matching = allCollections.collections.filter((c) =>
+      c.name.startsWith(prefix),
+    );
+
+    if (matching.length === 0) {
+      console.log(`[QdrantDocEmbeddings] No collections matching prefix ${prefix}`);
+      return;
+    }
+
+    for (const coll of matching) {
+      await this.client.deleteCollection(coll.name);
+      this.knownCollections.delete(coll.name);
+      console.log(`[QdrantDocEmbeddings] Deleted collection: ${coll.name}`);
+    }
+
+    console.log(
+      `[QdrantDocEmbeddings] Deleted ${matching.length} collection(s) for ${user}/${collection}`,
+    );
+  }
+}

ts/packages/flow/src/storage/embeddings/qdrant-graph.ts

@@ -0,0 +1,127 @@
+/**
+ * Qdrant graph embeddings write service.
+ *
+ * Stores entity/vector pairs in Qdrant for graph embeddings lookup.
+ * Collection naming: t_{user}_{collection}_{dimension}
+ * Collections are lazily created on first write with cosine distance.
+ *
+ * Python reference: trustgraph-flow/trustgraph/storage/graph_embeddings/qdrant/write.py
+ */
+
+import { QdrantClient } from "@qdrant/js-client-rest";
+import { randomUUID } from "node:crypto";
+import type { Term } from "@trustgraph/base";
+
+export interface QdrantGraphEmbeddingsConfig {
+  url?: string;
+  apiKey?: string;
+}
+
+export interface GraphEmbeddingEntity {
+  entity: Term;
+  vector: number[];
+  chunkId?: string;
+}
+
+export interface GraphEmbeddingsMessage {
+  user: string;
+  collection: string;
+  entities: GraphEmbeddingEntity[];
+}
+
+function getTermValue(term: Term): string | null {
+  switch (term.type) {
+    case "IRI":
+      return term.iri;
+    case "LITERAL":
+      return term.value;
+    case "BLANK":
+      return term.id;
+    case "TRIPLE":
+      return null;
+  }
+}
+
+export class QdrantGraphEmbeddingsStore {
+  private client: QdrantClient;
+  private knownCollections = new Set<string>();
+
+  constructor(config: QdrantGraphEmbeddingsConfig = {}) {
+    const url = config.url ?? process.env.QDRANT_URL ?? "http://localhost:6333";
+    const apiKey = config.apiKey ?? process.env.QDRANT_API_KEY;
+
+    this.client = new QdrantClient({ url, apiKey });
+
+    console.log("[QdrantGraphEmbeddings] Store initialized");
+  }
+
+  private collectionName(user: string, collection: string, dim: number): string {
+    return `t_${user}_${collection}_${dim}`;
+  }
+
+  private async ensureCollection(name: string, dim: number): Promise<void> {
+    if (this.knownCollections.has(name)) return;
+
+    const exists = await this.client.collectionExists(name);
+    if (!exists.exists) {
+      console.log(`[QdrantGraphEmbeddings] Creating collection ${name} (dim=${dim})`);
+      await this.client.createCollection(name, {
+        vectors: { size: dim, distance: "Cosine" },
+      });
+    }
+
+    this.knownCollections.add(name);
+  }
+
+  async store(message: GraphEmbeddingsMessage): Promise<void> {
+    for (const entry of message.entities) {
+      const entityValue = getTermValue(entry.entity);
+      if (!entityValue || entityValue === "") continue;
+      if (!entry.vector || entry.vector.length === 0) continue;
+
+      const dim = entry.vector.length;
+      const name = this.collectionName(message.user, message.collection, dim);
+
+      await this.ensureCollection(name, dim);
+
+      const payload: Record<string, unknown> = { entity: entityValue };
+      if (entry.chunkId) {
+        payload.chunk_id = entry.chunkId;
+      }
+
+      await this.client.upsert(name, {
+        points: [
+          {
+            id: randomUUID(),
+            vector: entry.vector,
+            payload,
+          },
+        ],
+      });
+    }
+  }
+
+  async deleteCollection(user: string, collection: string): Promise<void> {
+    const prefix = `t_${user}_${collection}_`;
+
+    const allCollections = await this.client.getCollections();
+    const matching = allCollections.collections.filter((c) =>
+      c.name.startsWith(prefix),
+    );
+
+    if (matching.length === 0) {
+      console.log(`[QdrantGraphEmbeddings] No collections matching prefix ${prefix}`);
+      return;
+    }
+
+    for (const coll of matching) {
+      await this.client.deleteCollection(coll.name);
+      this.knownCollections.delete(coll.name);
+      console.log(`[QdrantGraphEmbeddings] Deleted collection: ${coll.name}`);
+    }
+
+    console.log(
+      `[QdrantGraphEmbeddings] Deleted ${matching.length} collection(s) for ${user}/${collection}`,
+    );
+  }
+}