Feature/librarian (#307)

* Bring QDrant up-to-date

* Tables for data from queue outputs

- Pass single Pulsar client to everything in gateway & librarian
- Pulsar listener-name support in gateway
- PDF and text load working in librarian

* Complete Cassandra schema

* Add librarian support to templates

templates/components.jsonnet

@@ -34,6 +34,9 @@
    "graph-rag": import "components/graph-rag.jsonnet",
    "document-rag": import "components/document-rag.jsonnet",
 
+   // Librarian - document management
+   "librarian": import "components/librarian.jsonnet",
+
    // Vector stores
    "vector-store-milvus": import "components/milvus.jsonnet",
    "vector-store-qdrant": import "components/qdrant.jsonnet",

templates/components/librarian.jsonnet

@@ -0,0 +1,43 @@
+local base = import "base/base.jsonnet";
+local images = import "values/images.jsonnet";
+local url = import "values/url.jsonnet";
+local minio = import "stores/minio.jsonnet";
+local cassandra = import "stores/cassandra.jsonnet";
+
+{
+
+    "librarian" +: {
+    
+        create:: function(engine)
+
+            local container =
+                engine.container("librarian")
+                    .with_image(images.trustgraph_flow)
+                    .with_command([
+                        "librarian",
+                        "-p",
+                        url.pulsar,
+                    ])
+                    .with_limits("0.5", "256M")
+                    .with_reservations("0.1", "256M");
+
+            local containerSet = engine.containers(
+                "librarian", [ container ]
+            );
+
+            local service =
+                engine.internalService(containerSet)
+                .with_port(8000, 8000, "metrics");
+
+            engine.resources([
+                containerSet,
+                service,
+            ])
+
+    },
+
+}
+
+    // Minio and Cassandra are used by the Librarian
+    + minio + cassandra
+

templates/components/trustgraph.jsonnet

@@ -1,8 +1,6 @@
 local base = import "base/base.jsonnet";
 local images = import "values/images.jsonnet";
 local url = import "values/url.jsonnet";
-local minio = import "stores/minio.jsonnet";
-local cassandra = import "stores/cassandra.jsonnet";
 
 {
 
@@ -184,6 +182,3 @@ local cassandra = import "stores/cassandra.jsonnet";
 
 }
 
-    // Minio and Cassandra are used by the Librarian
-    + minio + cassandra
-

templates/values/images.jsonnet

@@ -14,7 +14,7 @@ local version = import "version.jsonnet";
     trustgraph_bedrock: "docker.io/trustgraph/trustgraph-bedrock:" + version,
     trustgraph_vertexai: "docker.io/trustgraph/trustgraph-vertexai:" + version,
     trustgraph_hf: "docker.io/trustgraph/trustgraph-hf:" + version,
-    qdrant: "docker.io/qdrant/qdrant:v1.11.1",
+    qdrant: "docker.io/qdrant/qdrant:v1.13.3",
     memgraph_mage: "docker.io/memgraph/memgraph-mage:1.22-memgraph-2.22",
     memgraph_lab: "docker.io/memgraph/lab:2.19.1",
     falkordb: "docker.io/falkordb/falkordb:latest",