Merge branch 'release/v0.21' into pulsar-api-support

.github/workflows/release.yaml

@@ -5,17 +5,17 @@ on:
   workflow_dispatch:
   push:
     tags:
-      - v0.18.*
+      - v*
 
 permissions:
   contents: read
 
 jobs:
 
-  deploy:
+  python-packages:
 
-    name: Build everything
+    name: Release Python packages
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     permissions:
       contents: write
       id-token: write
@@ -25,50 +25,73 @@ jobs:
     steps:
 
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
-
-      - name: Log in to Docker Hub
-        uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a
-        with:
-          username: ${{ vars.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_SECRET }}
-
-      - name: Install build dependencies
-        run: pip3 install jsonnet
 
       - name: Get version
         id: version
         run: echo VERSION=$(git describe --exact-match --tags | sed 's/^v//') >> $GITHUB_OUTPUT
 
-      - run: echo ${{ steps.version.outputs.VERSION }}
-
       - name: Build packages
         run: make packages VERSION=${{ steps.version.outputs.VERSION }}
 
       - name: Publish release distributions to PyPI
         uses: pypa/gh-action-pypi-publish@release/v1
 
-      - name: Extract metadata for container
+  deploy-container-image:
-        id: meta
-        uses: docker/metadata-action@v4
-        with:
-          images: trustgraph/trustgraph-flow
-          tags: |
-            type=ref,event=branch
-            type=ref,event=pr
-            type=semver,pattern={{version}}
-            type=semver,pattern={{major}}.{{minor}}
-            type=sha
 
-      - name: Build and push Docker image
+    name: Release container image
-        id: push
+    runs-on: ubuntu-24.04
-        uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671
+    permissions:
-        with:
+      contents: write
-          context: .
+      id-token: write
-          file: ./Containerfile
+    environment:
-          push: true
+      name: release
-          tags: ${{ steps.meta.outputs.tags }}
+
-          labels: ${{ steps.meta.outputs.labels }}
+    steps:
+
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Docker Hub token
+        run: echo ${{ secrets.DOCKER_SECRET }} > docker-token.txt
+
+      - name: Authenticate with Docker hub
+        run: make docker-hub-login
+
+      - name: Get version
+        id: version
+        run: echo VERSION=$(git describe --exact-match --tags | sed 's/^v//') >> $GITHUB_OUTPUT
+
+      - name: Put version into package manifests
+        run: make update-package-versions VERSION=${{ steps.version.outputs.VERSION }}
+
+      - name: Build containers
+        run: make container VERSION=${{ steps.version.outputs.VERSION }}
+
+      - name: Push containers
+        run: make push VERSION=${{ steps.version.outputs.VERSION }}
+
+  release-bundle:
+
+    name: Upload release bundle
+    runs-on: ubuntu-24.04
+    permissions:
+      contents: write
+      id-token: write
+    environment:
+      name: release
+
+    steps:
+
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install build dependencies
+        run: pip3 install jsonnet
+
+      - name: Get version
+        id: version
+        run: echo VERSION=$(git describe --exact-match --tags | sed 's/^v//') >> $GITHUB_OUTPUT
 
       - name: Create deploy bundle
         run: templates/generate-all deploy.zip ${{ steps.version.outputs.VERSION }}

Containerfile

@@ -11,15 +11,26 @@ ENV PIP_BREAK_SYSTEM_PACKAGES=1
 RUN dnf install -y python3 python3-pip python3-wheel python3-aiohttp \
     python3-rdflib
 
-RUN pip3 install torch --index-url https://download.pytorch.org/whl/cpu
+RUN pip3 install torch==2.5.1+cpu \
+    --index-url https://download.pytorch.org/whl/cpu
 
-RUN pip3 install anthropic boto3 cohere openai google-cloud-aiplatform ollama google-generativeai \
+RUN pip3 install \
-    langchain langchain-core langchain-huggingface langchain-text-splitters \
+    anthropic boto3 cohere openai google-cloud-aiplatform \
-    langchain-community pymilvus sentence-transformers transformers \
+    ollama google-generativeai \
-    huggingface-hub pulsar-client cassandra-driver pyyaml \
+    langchain==0.3.13 langchain-core==0.3.28 langchain-huggingface==0.1.2 \
+    langchain-text-splitters==0.3.4 \
+    langchain-community==0.3.13 \
+    sentence-transformers==3.4.0 transformers==4.47.1 \
+    huggingface-hub==0.27.0 \
+    pymilvus \
+    pulsar-client==3.5.0 cassandra-driver pyyaml \
     neo4j tiktoken falkordb && \
     pip3 cache purge
 
+# Most commonly used embeddings model, just build it into the container
+# image
+RUN huggingface-cli download sentence-transformers/all-MiniLM-L6-v2
+
 # ----------------------------------------------------------------------------
 # Build a container which contains the built Python packages.  The build
 # creates a bunch of left-over cruft, a separate phase means this is only

Makefile

@@ -30,7 +30,7 @@ packages: update-package-versions
 pypi-upload:
 	twine upload dist/*-${VERSION}.*
 
-CONTAINER=docker.io/trustgraph/trustgraph-flow
+CONTAINER_BASE=docker.io/trustgraph
 
 update-package-versions:
 	mkdir -p trustgraph-cli/trustgraph
@@ -44,11 +44,23 @@ update-package-versions:
 	echo __version__ = \"${VERSION}\" > trustgraph/trustgraph/trustgraph_version.py
 
 container: update-package-versions
-	${DOCKER} build -f Containerfile -t ${CONTAINER}:${VERSION} \
+	${DOCKER} build -f containers/Containerfile.base \
-	    --format docker
+	    -t ${CONTAINER_BASE}/trustgraph-base:${VERSION} .
+	${DOCKER} build -f containers/Containerfile.flow \
+	    -t ${CONTAINER_BASE}/trustgraph-flow:${VERSION} .
+	${DOCKER} build -f containers/Containerfile.bedrock \
+	    -t ${CONTAINER_BASE}/trustgraph-bedrock:${VERSION} .
+	${DOCKER} build -f containers/Containerfile.vertexai \
+	    -t ${CONTAINER_BASE}/trustgraph-vertexai:${VERSION} .
+	${DOCKER} build -f containers/Containerfile.hf \
+	    -t ${CONTAINER_BASE}/trustgraph-hf:${VERSION} .
 
 push:
-	${DOCKER} push ${CONTAINER}:${VERSION}
+	${DOCKER} push ${CONTAINER_BASE}/trustgraph-base:${VERSION}
+	${DOCKER} push ${CONTAINER_BASE}/trustgraph-flow:${VERSION}
+	${DOCKER} push ${CONTAINER_BASE}/trustgraph-bedrock:${VERSION}
+	${DOCKER} push ${CONTAINER_BASE}/trustgraph-vertexai:${VERSION}
+	${DOCKER} push ${CONTAINER_BASE}/trustgraph-hf:${VERSION}
 
 clean:
 	rm -rf wheels/
@@ -104,5 +116,5 @@ update-dcs: set-version
 
 docker-hub-login:
 	cat docker-token.txt | \
-	    docker login -u trustgraph --password-stdin registry-1.docker.io
+	    ${DOCKER} login -u trustgraph --password-stdin registry-1.docker.io
 

containers/Containerfile.base

@@ -0,0 +1,48 @@
+
+# ----------------------------------------------------------------------------
+# Build an AI container.  This does the torch install which is huge, and I
+# like to avoid re-doing this.
+# ----------------------------------------------------------------------------
+
+FROM docker.io/fedora:40 AS base
+
+ENV PIP_BREAK_SYSTEM_PACKAGES=1
+
+RUN dnf install -y python3 python3-pip python3-wheel python3-aiohttp && \
+    dnf clean all
+
+RUN pip3 install --no-cache-dir pulsar-client==3.5.0
+
+# ----------------------------------------------------------------------------
+# Build a container which contains the built Python packages.  The build
+# creates a bunch of left-over cruft, a separate phase means this is only
+# needed to support package build
+# ----------------------------------------------------------------------------
+
+FROM base AS build
+
+COPY trustgraph-base/ /root/build/trustgraph-base/
+COPY trustgraph-cli/ /root/build/trustgraph-cli/
+
+WORKDIR /root/build/
+
+RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-base/
+RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-cli/
+
+# ----------------------------------------------------------------------------
+# Finally, the target container.  Start with base and add the package.
+# ----------------------------------------------------------------------------
+
+FROM base
+
+COPY --from=build /root/wheels /root/wheels
+
+RUN \
+    pip3 install --no-cache-dir /root/wheels/trustgraph_base-* && \
+    pip3 install --no-cache-dir /root/wheels/trustgraph_cli-* && \
+    rm -rf /root/wheels
+
+WORKDIR /
+
+
+

containers/Containerfile.bedrock

@@ -0,0 +1,48 @@
+
+# ----------------------------------------------------------------------------
+# Build an AI container.  This does the torch install which is huge, and I
+# like to avoid re-doing this.
+# ----------------------------------------------------------------------------
+
+FROM docker.io/fedora:40 AS base
+
+ENV PIP_BREAK_SYSTEM_PACKAGES=1
+
+RUN dnf install -y python3 python3-pip python3-wheel python3-aiohttp \
+    python3-rdflib
+
+RUN pip3 install --no-cache-dir boto3 pulsar-client==3.5.0
+
+# ----------------------------------------------------------------------------
+# Build a container which contains the built Python packages.  The build
+# creates a bunch of left-over cruft, a separate phase means this is only
+# needed to support package build
+# ----------------------------------------------------------------------------
+
+FROM base AS build
+
+COPY trustgraph-base/ /root/build/trustgraph-base/
+COPY trustgraph-bedrock/ /root/build/trustgraph-bedrock/
+
+WORKDIR /root/build/
+
+RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-base/
+RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-bedrock/
+
+RUN ls /root/wheels
+
+# ----------------------------------------------------------------------------
+# Finally, the target container.  Start with base and add the package.
+# ----------------------------------------------------------------------------
+
+FROM base
+
+COPY --from=build /root/wheels /root/wheels
+
+RUN \
+    pip3 install --no-cache-dir /root/wheels/trustgraph_base-* && \
+    pip3 install --no-cache-dir /root/wheels/trustgraph_bedrock-* && \
+    rm -rf /root/wheels
+
+WORKDIR /
+

containers/Containerfile.flow

@@ -0,0 +1,60 @@
+
+# ----------------------------------------------------------------------------
+# Build an AI container.  This does the torch install which is huge, and I
+# like to avoid re-doing this.
+# ----------------------------------------------------------------------------
+
+FROM docker.io/fedora:40 AS base
+
+ENV PIP_BREAK_SYSTEM_PACKAGES=1
+
+RUN dnf install -y python3 python3-pip python3-wheel python3-aiohttp \
+    python3-rdflib
+
+RUN pip3 install --no-cache-dir \
+    anthropic cohere openai google-generativeai \
+    ollama \
+    langchain==0.3.13 langchain-core==0.3.28 \
+    langchain-text-splitters==0.3.4 \
+    langchain-community==0.3.13 \
+    pymilvus \
+    pulsar-client==3.5.0 cassandra-driver pyyaml \
+    neo4j tiktoken falkordb && \
+    pip3 cache purge
+
+# ----------------------------------------------------------------------------
+# Build a container which contains the built Python packages.  The build
+# creates a bunch of left-over cruft, a separate phase means this is only
+# needed to support package build
+# ----------------------------------------------------------------------------
+
+FROM base AS build
+
+COPY trustgraph-base/ /root/build/trustgraph-base/
+COPY trustgraph-flow/ /root/build/trustgraph-flow/
+COPY trustgraph-cli/ /root/build/trustgraph-cli/
+
+WORKDIR /root/build/
+
+RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-base/
+RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-flow/
+RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-cli/
+
+RUN ls /root/wheels
+
+# ----------------------------------------------------------------------------
+# Finally, the target container.  Start with base and add the package.
+# ----------------------------------------------------------------------------
+
+FROM base
+
+COPY --from=build /root/wheels /root/wheels
+
+RUN \
+    pip3 install --no-cache-dir /root/wheels/trustgraph_base-* && \
+    pip3 install --no-cache-dir /root/wheels/trustgraph_flow-* && \
+    pip3 install --no-cache-dir /root/wheels/trustgraph_cli-* && \
+    rm -rf /root/wheels
+
+WORKDIR /
+

containers/Containerfile.hf

@@ -0,0 +1,75 @@
+
+# ----------------------------------------------------------------------------
+# Build an AI container.  This does the torch install which is huge, and I
+# like to avoid re-doing this.
+# ----------------------------------------------------------------------------
+
+FROM docker.io/fedora:40 AS ai
+
+ENV PIP_BREAK_SYSTEM_PACKAGES=1
+
+RUN dnf install -y python3 python3-pip python3-wheel python3-aiohttp \
+    python3-rdflib
+
+RUN pip3 install torch==2.5.1+cpu \
+    --index-url https://download.pytorch.org/whl/cpu
+
+RUN pip3 install --no-cache-dir \
+    langchain==0.3.13 langchain-core==0.3.28 langchain-huggingface==0.1.2 \
+    langchain-community==0.3.13 \
+    sentence-transformers==3.4.0 transformers==4.47.1 \
+    huggingface-hub==0.27.0 \
+    pulsar-client==3.5.0
+
+# Most commonly used embeddings model, just build it into the container
+# image
+RUN huggingface-cli download sentence-transformers/all-MiniLM-L6-v2
+
+# ----------------------------------------------------------------------------
+# Build a container which contains the built Python packages.  The build
+# creates a bunch of left-over cruft, a separate phase means this is only
+# needed to support package build
+# ----------------------------------------------------------------------------
+
+FROM ai AS build
+
+COPY trustgraph-base/ /root/build/trustgraph-base/
+COPY trustgraph-flow/ /root/build/trustgraph-flow/
+COPY trustgraph-vertexai/ /root/build/trustgraph-vertexai/
+COPY trustgraph-bedrock/ /root/build/trustgraph-bedrock/
+COPY trustgraph-embeddings-hf/ /root/build/trustgraph-embeddings-hf/
+COPY trustgraph-cli/ /root/build/trustgraph-cli/
+
+WORKDIR /root/build/
+
+RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-base/
+RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-flow/
+RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-vertexai/
+RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-bedrock/
+RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-embeddings-hf/
+RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-cli/
+
+RUN ls /root/wheels
+
+# ----------------------------------------------------------------------------
+# Finally, the target container.  Start with base and add the package.
+# ----------------------------------------------------------------------------
+
+FROM ai
+
+COPY --from=build /root/wheels /root/wheels
+
+RUN \
+    pip3 install /root/wheels/trustgraph_base-* && \
+    pip3 install /root/wheels/trustgraph_flow-* && \
+    pip3 install /root/wheels/trustgraph_vertexai-* && \
+    pip3 install /root/wheels/trustgraph_bedrock-* && \
+    pip3 install /root/wheels/trustgraph_embeddings_hf-* && \
+    pip3 install /root/wheels/trustgraph_cli-* && \
+    pip3 cache purge && \
+    rm -rf /root/wheels
+
+WORKDIR /
+
+CMD sleep 1000000
+

containers/Containerfile.vertexai

@@ -0,0 +1,51 @@
+
+# ----------------------------------------------------------------------------
+# Build an AI container.  This does the torch install which is huge, and I
+# like to avoid re-doing this.
+# ----------------------------------------------------------------------------
+
+FROM docker.io/fedora:40 AS base
+
+ENV PIP_BREAK_SYSTEM_PACKAGES=1
+
+RUN dnf install -y python3 python3-pip python3-wheel python3-aiohttp \
+    python3-rdflib
+
+RUN pip3 install --no-cache-dir \
+    google-cloud-aiplatform pulsar-client==3.5.0
+
+# ----------------------------------------------------------------------------
+# Build a container which contains the built Python packages.  The build
+# creates a bunch of left-over cruft, a separate phase means this is only
+# needed to support package build
+# ----------------------------------------------------------------------------
+
+FROM base AS build
+
+COPY trustgraph-base/ /root/build/trustgraph-base/
+COPY trustgraph-vertexai/ /root/build/trustgraph-vertexai/
+
+WORKDIR /root/build/
+
+RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-base/
+RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-vertexai/
+
+RUN ls /root/wheels
+
+# ----------------------------------------------------------------------------
+# Finally, the target container.  Start with base and add the package.
+# ----------------------------------------------------------------------------
+
+FROM base
+
+COPY --from=build /root/wheels /root/wheels
+
+RUN \
+    pip3 install --no-cache-dir /root/wheels/trustgraph_base-* && \
+    pip3 install --no-cache-dir /root/wheels/trustgraph_vertexai-* && \
+    rm -rf /root/wheels
+
+WORKDIR /
+
+
+

docs/README.quickstart-docker-compose.md

@@ -113,8 +113,9 @@ Choose one of the `Docker Compose` files that meets your preferred model and gra
 ### AWS Bedrock API
 
 ```
-export AWS_ID_KEY=<ID-KEY-HERE>
+export AWS_ACCESS_KEY_ID=<ID-KEY-HERE>
-export AWS_SECRET_KEY=<TOKEN-GOES-HERE>
+export AWS_SECRET_ACCESS_KEY=<TOKEN-GOES-HERE>
+export AWS_DEFAULT_REGION=<REGION-HERE>
 docker compose -f tg-launch-bedrock-cassandra.yaml up -d # Using Cassandra as the graph store
 docker compose -f tg-launch-bedrock-neo4j.yaml up -d # Using Neo4j as the graph store
 ```

grafana/dashboards/dashboard.json

@@ -577,7 +577,7 @@
           "disableTextWrap": false,
           "editorMode": "builder",
           "exemplar": false,
-          "expr": "increase(processing_count_total{status!=\"success\"}[$__rate_interval])",
+          "expr": "sum by(job) (increase(rate_limit_count_total[$__rate_interval]))",
           "format": "time_series",
           "fullMetaSearch": false,
           "includeNullMetadata": true,
@@ -588,7 +588,7 @@
           "useBackend": false
         }
       ],
-      "title": "Errors",
+      "title": "Rate limit events",
       "type": "timeseries"
     },
     {

prometheus/prometheus.yml

@@ -20,6 +20,18 @@ scrape_configs:
       - targets:
         - 'pulsar:8080'
 
+  - job_name: 'bookie'
+    scrape_interval: 5s
+    static_configs:
+      - targets:
+        - 'bookie:8000'
+
+  - job_name: 'zookeeper'
+    scrape_interval: 5s
+    static_configs:
+      - targets:
+        - 'zookeeper:8000'
+
   - job_name: 'pdf-decoder'
     scrape_interval: 5s
     static_configs:
@@ -32,11 +44,17 @@ scrape_configs:
       - targets:
         - 'chunker:8000'
 
-  - job_name: 'vectorize'
+  - job_name: 'document-embeddings'
     scrape_interval: 5s
     static_configs:
       - targets:
-        - 'vectorize:8000'
+        - 'document-embeddings:8000'
+
+  - job_name: 'graph-embeddings'
+    scrape_interval: 5s
+    static_configs:
+      - targets:
+        - 'graph-embeddings:8000'
 
   - job_name: 'embeddings'
     scrape_interval: 5s
@@ -74,6 +92,12 @@ scrape_configs:
       - targets:
         - 'metering-rag:8000'
 
+  - job_name: 'store-doc-embeddings'
+    scrape_interval: 5s
+    static_configs:
+      - targets:
+        - 'store-doc-embeddings:8000'
+
   - job_name: 'store-graph-embeddings'
     scrape_interval: 5s
     static_configs:
@@ -104,6 +128,12 @@ scrape_configs:
       - targets:
         - 'graph-rag:8000'
 
+  - job_name: 'document-rag'
+    scrape_interval: 5s
+    static_configs:
+      - targets:
+        - 'document-rag:8000'
+
   - job_name: 'prompt'
     scrape_interval: 5s
     static_configs:
@@ -122,6 +152,12 @@ scrape_configs:
       - targets:
         - 'query-graph-embeddings:8000'
 
+  - job_name: 'query-doc-embeddings'
+    scrape_interval: 5s
+    static_configs:
+      - targets:
+        - 'query-doc-embeddings:8000'
+
   - job_name: 'query-triples'
     scrape_interval: 5s
     static_configs:
@@ -145,3 +181,7 @@ scrape_configs:
     static_configs:
       - targets:
         - 'workbench-ui:8000'
+
+# Cassandra
+# qdrant
+

templates/components.jsonnet

@@ -1,47 +1,70 @@
 {
+
+   // Essentials
+   "trustgraph-base": import "components/trustgraph.jsonnet",
+   "pulsar": import "components/pulsar.jsonnet",
+
+   // LLMs
    "azure": import "components/azure.jsonnet",
    "azure-openai": import "components/azure-openai.jsonnet",
    "bedrock": import "components/bedrock.jsonnet",
    "claude": import "components/claude.jsonnet",
    "cohere": import "components/cohere.jsonnet",
-   "document-rag": import "components/document-rag.jsonnet",
-   "embeddings-hf": import "components/embeddings-hf.jsonnet",
-   "embeddings-ollama": import "components/embeddings-ollama.jsonnet",
    "googleaistudio": import "components/googleaistudio.jsonnet",
-   "grafana": import "components/grafana.jsonnet",
+   "ollama": import "components/ollama.jsonnet",
+   "openai": import "components/openai.jsonnet",
+   "vertexai": import "components/vertexai.jsonnet",
+
+   // LLMs for RAG
+   "azure-rag": import "components/azure-rag.jsonnet",
+   "azure-openai-rag": import "components/azure-openai-rag.jsonnet",
+   "bedrock-rag": import "components/bedrock-rag.jsonnet",
+   "claude-rag": import "components/claude-rag.jsonnet",
+   "cohere-rag": import "components/cohere-rag.jsonnet",
+   "googleaistudio-rag": import "components/googleaistudio-rag.jsonnet",
+   "ollama-rag": import "components/ollama-rag.jsonnet",
+   "openai-rag": import "components/openai-rag.jsonnet",
+   "vertexai-rag": import "components/vertexai-rag.jsonnet",
+
+   // Embeddings
+   "embeddings-ollama": import "components/embeddings-ollama.jsonnet",
+   "embeddings-hf": import "components/embeddings-hf.jsonnet",
+
+   // Processing pipelines
    "graph-rag": import "components/graph-rag.jsonnet",
+   "document-rag": import "components/document-rag.jsonnet",
+
+   // Vector stores
+   "vector-store-milvus": import "components/milvus.jsonnet",
+   "vector-store-qdrant": import "components/qdrant.jsonnet",
+   "vector-store-pinecone": import "components/pinecone.jsonnet",
+
+   // Triples stores
    "triple-store-cassandra": import "components/cassandra.jsonnet",
    "triple-store-neo4j": import "components/neo4j.jsonnet",
    "triple-store-falkordb": import "components/falkordb.jsonnet",
    "triple-store-memgraph": import "components/memgraph.jsonnet",
+
+   // Observability support
+   "grafana": import "components/grafana.jsonnet",
+
+   // Pulsar manager is a UI for Pulsar.  Uses a LOT of memory
+   "pulsar-manager": import "components/pulsar-manager.jsonnet",
+
    "llamafile": import "components/llamafile.jsonnet",
-   "ollama": import "components/ollama.jsonnet",
-   "openai": import "components/openai.jsonnet",
    "override-recursive-chunker": import "components/chunker-recursive.jsonnet",
 
+   // The prompt manager
    "prompt-template": import "components/prompt-template.jsonnet",
    "prompt-overrides": import "components/prompt-overrides.jsonnet",
 
-   "pulsar": import "components/pulsar.jsonnet",
+   // ReAct agent
-   "pulsar-manager": import "components/pulsar-manager.jsonnet",
-   "trustgraph-base": import "components/trustgraph.jsonnet",
-   "vector-store-milvus": import "components/milvus.jsonnet",
-   "vector-store-qdrant": import "components/qdrant.jsonnet",
-   "vector-store-pinecone": import "components/pinecone.jsonnet",
-   "vertexai": import "components/vertexai.jsonnet",
-   "workbench-ui": import "components/workbench-ui.jsonnet",
-   "null": {},
-
    "agent-manager-react": import "components/agent-manager-react.jsonnet",
 
-   // FIXME: Dupes
+   // Optional UI
-   "cassandra": import "components/cassandra.jsonnet",
+   "workbench-ui": import "components/workbench-ui.jsonnet",
-   "neo4j": import "components/neo4j.jsonnet",
+
-   "memgraph": import "components/memgraph.jsonnet",
+   // Does nothing.  But, can be a hack to overwrite parameters
-   "qdrant": import "components/qdrant.jsonnet",
+   "null": {},
-   "pinecone": import "components/pinecone.jsonnet",
-   "milvus": import "components/milvus.jsonnet",
-   "falkordb": import "components/falkordb.jsonnet",
-   "trustgraph": import "components/trustgraph.jsonnet",
 
 }

templates/components/agent-manager-react.jsonnet

@@ -14,11 +14,15 @@ local default_prompts = import "prompts/default-prompts.jsonnet";
 
             local container =
                 engine.container("agent-manager")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "agent-manager-react",
                         "-p",
                         url.pulsar,
+                        "--prompt-request-queue",
+                        "non-persistent://tg/request/prompt-rag",
+                        "--prompt-response-queue",
+                        "non-persistent://tg/response/prompt-rag",
                         "--tool-type",
                     ] + [
                         tool.id + "=" + tool.type

templates/components/azure-openai-rag.jsonnet

@@ -0,0 +1,61 @@
+local base = import "base/base.jsonnet";
+local images = import "values/images.jsonnet";
+local url = import "values/url.jsonnet";
+local prompts = import "prompts/mixtral.jsonnet";
+
+{
+
+    with:: function(key, value)
+        self + {
+            ["ollama-rag-" + key]:: value,
+        },
+
+    "azure-openai-rag-model":: "GPT-3.5-Turbo",
+    "azure-openai-rag-max-output-tokens":: 4192,
+    "azure-openai-rag-temperature":: 0.0,
+
+    "text-completion-rag" +: {
+    
+        create:: function(engine)
+
+            local envSecrets = engine.envSecrets("azure-openai-credentials")
+                .with_env_var("AZURE_TOKEN", "azure-token");
+
+            local containerRag =
+                engine.container("text-completion-rag")
+                    .with_image(images.trustgraph_flow)
+                    .with_command([
+                        "text-completion-azure",
+                        "-p",
+                        url.pulsar,
+                        "-x",
+                        std.toString($["azure-openai-rag-max-output-tokens"]),
+                        "-t",
+                        "%0.3f" % $["azure-openai-rag-temperature"],
+                        "-i",
+                        "non-persistent://tg/request/text-completion-rag",
+                        "-o",
+                        "non-persistent://tg/response/text-completion-rag",
+                    ])
+                    .with_env_var_secrets(envSecrets)
+                    .with_limits("0.5", "128M")
+                    .with_reservations("0.1", "128M");
+
+            local containerSetRag = engine.containers(
+                "text-completion-rag", [ containerRag ]
+            );
+
+            local serviceRag =
+                engine.internalService(containerSetRag)
+                .with_port(8000, 8000, "metrics");
+
+            engine.resources([
+                envSecrets,
+                containerSetRag,
+                serviceRag,
+            ])
+
+    },
+
+} + prompts
+

templates/components/azure-openai.jsonnet

@@ -5,6 +5,11 @@ local prompts = import "prompts/mixtral.jsonnet";
 
 {
 
+    with:: function(key, value)
+        self + {
+            ["azure-openai-" + key]:: value,
+        },
+
     "azure-openai-model":: "GPT-3.5-Turbo",
     "azure-openai-max-output-tokens":: 4192,
     "azure-openai-temperature":: 0.0,
@@ -18,7 +23,7 @@ local prompts = import "prompts/mixtral.jsonnet";
 
             local container =
                 engine.container("text-completion")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "text-completion-azure-openai",
                         "-p",
@@ -34,48 +39,18 @@ local prompts = import "prompts/mixtral.jsonnet";
                     .with_limits("0.5", "128M")
                     .with_reservations("0.1", "128M");
 
-            local containerRag =
-                engine.container("text-completion-rag")
-                    .with_image(images.trustgraph)
-                    .with_command([
-                        "text-completion-azure",
-                        "-p",
-                        url.pulsar,
-                        "-x",
-                        std.toString($["azure-openai-max-output-tokens"]),
-                        "-t",
-                        "%0.3f" % $["azure-openai-temperature"],
-                        "-i",
-                        "non-persistent://tg/request/text-completion-rag",
-                        "-o",
-                        "non-persistent://tg/response/text-completion-rag",
-                    ])
-                    .with_env_var_secrets(envSecrets)
-                    .with_limits("0.5", "128M")
-                    .with_reservations("0.1", "128M");
-
             local containerSet = engine.containers(
                 "text-completion", [ container ]
             );
 
-            local containerSetRag = engine.containers(
-                "text-completion-rag", [ containerRag ]
-            );
-
             local service =
                 engine.internalService(containerSet)
                 .with_port(8000, 8000, "metrics");
 
-            local serviceRag =
-                engine.internalService(containerSetRag)
-                .with_port(8000, 8000, "metrics");
-
             engine.resources([
                 envSecrets,
                 containerSet,
-                containerSetRag,
                 service,
-                serviceRag,
             ])
 
     },

templates/components/azure-rag.jsonnet

@@ -0,0 +1,60 @@
+local images = import "values/images.jsonnet";
+local url = import "values/url.jsonnet";
+local prompts = import "prompts/mixtral.jsonnet";
+
+{
+
+    with:: function(key, value)
+        self + {
+            ["azure-rag-" + key]:: value,
+        },
+
+    "azure-rag-max-output-tokens":: 4096,
+    "azure-rag-temperature":: 0.0,
+
+    "text-completion-rag" +: {
+    
+        create:: function(engine)
+
+            local envSecrets = engine.envSecrets("azure-credentials")
+                .with_env_var("AZURE_TOKEN", "azure-token")
+                .with_env_var("AZURE_ENDPOINT", "azure-endpoint");
+
+            local containerRag =
+                engine.container("text-completion-rag")
+                    .with_image(images.trustgraph_flow)
+                    .with_command([
+                        "text-completion-azure",
+                        "-p",
+                        url.pulsar,
+                        "-x",
+                        std.toString($["azure-rag-max-output-tokens"]),
+                        "-t",
+                        "%0.3f" % $["azure-rag-temperature"],
+                        "-i",
+                        "non-persistent://tg/request/text-completion-rag",
+                        "-o",
+                        "non-persistent://tg/response/text-completion-rag",
+                    ])
+                    .with_env_var_secrets(envSecrets)
+                    .with_limits("0.5", "128M")
+                    .with_reservations("0.1", "128M");
+
+            local containerSetRag = engine.containers(
+                "text-completion-rag", [ containerRag ]
+            );
+
+            local serviceRag =
+                engine.internalService(containerSetRag)
+                .with_port(8000, 8000, "metrics");
+
+            engine.resources([
+                envSecrets,
+                containerSetRag,
+                serviceRag,
+            ])
+
+    }
+
+} + prompts
+

templates/components/azure.jsonnet

@@ -1,10 +1,14 @@
-local base = import "base/base.jsonnet";
 local images = import "values/images.jsonnet";
 local url = import "values/url.jsonnet";
 local prompts = import "prompts/mixtral.jsonnet";
 
 {
 
+    with:: function(key, value)
+        self + {
+            ["azure-" + key]:: value,
+        },
+
     "azure-max-output-tokens":: 4096,
     "azure-temperature":: 0.0,
 
@@ -18,7 +22,7 @@ local prompts = import "prompts/mixtral.jsonnet";
 
             local container =
                 engine.container("text-completion")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "text-completion-azure",
                         "-p",
@@ -32,48 +36,18 @@ local prompts = import "prompts/mixtral.jsonnet";
                     .with_limits("0.5", "128M")
                     .with_reservations("0.1", "128M");
 
-            local containerRag =
-                engine.container("text-completion-rag")
-                    .with_image(images.trustgraph)
-                    .with_command([
-                        "text-completion-azure",
-                        "-p",
-                        url.pulsar,
-                        "-x",
-                        std.toString($["azure-max-output-tokens"]),
-                        "-t",
-                        "%0.3f" % $["azure-temperature"],
-                        "-i",
-                        "non-persistent://tg/request/text-completion-rag",
-                        "-o",
-                        "non-persistent://tg/response/text-completion-rag",
-                    ])
-                    .with_env_var_secrets(envSecrets)
-                    .with_limits("0.5", "128M")
-                    .with_reservations("0.1", "128M");
-
             local containerSet = engine.containers(
                 "text-completion", [ container ]
             );
 
-            local containerSetRag = engine.containers(
-                "text-completion-rag", [ containerRag ]
-            );
-
             local service =
                 engine.internalService(containerSet)
                 .with_port(8000, 8000, "metrics");
 
-            local serviceRag =
-                engine.internalService(containerSetRag)
-                .with_port(8000, 8000, "metrics");
-
             engine.resources([
                 envSecrets,
                 containerSet,
-                containerSetRag,
                 service,
-                serviceRag,
             ])
 
     }

templates/components/bedrock-rag.jsonnet

@@ -0,0 +1,66 @@
+local base = import "base/base.jsonnet";
+local images = import "values/images.jsonnet";
+local url = import "values/url.jsonnet";
+local prompts = import "prompts/mixtral.jsonnet";
+local chunker = import "chunker-recursive.jsonnet";
+
+{
+
+    with:: function(key, value)
+        self + {
+            ["bedrock-rag-" + key]:: value,
+        },
+
+    "bedrock-rag-max-output-tokens":: 4096,
+    "bedrock-rag-temperature":: 0.0,
+    "bedrock-rag-model":: "mistral.mixtral-8x7b-instruct-v0:1",
+
+    "text-completion-rag" +: {
+    
+        create:: function(engine)
+
+            local envSecrets = engine.envSecrets("bedrock-credentials")
+                .with_env_var("AWS_ACCESS_KEY_ID", "aws-id-key")
+                .with_env_var("AWS_SECRET_ACCESS_KEY", "aws-secret")
+                .with_env_var("AWS_DEFAULT_REGION", "aws-region");
+
+            local containerRag =
+                engine.container("text-completion-rag")
+                    .with_image(images.trustgraph_bedrock)
+                    .with_command([
+                        "text-completion-bedrock",
+                        "-p",
+                        url.pulsar,
+                        "-x",
+                        std.toString($["bedrock-rag-max-output-tokens"]),
+                        "-t",
+                        "%0.3f" % $["bedrock-rag-temperature"],
+                        "-m",
+                        $["bedrock-rag-model"],
+                        "-i",
+                        "non-persistent://tg/request/text-completion-rag",
+                        "-o",
+                        "non-persistent://tg/response/text-completion-rag",
+              	    ])
+                    .with_env_var_secrets(envSecrets)
+                    .with_limits("0.5", "128M")
+                    .with_reservations("0.1", "128M");
+
+            local containerSetRag = engine.containers(
+                "text-completion-rag", [ containerRag ]
+            );
+
+            local serviceRag =
+                engine.internalService(containerSetRag)
+                .with_port(8000, 8000, "metrics");
+
+            engine.resources([
+                envSecrets,
+                containerSetRag,
+                serviceRag,
+            ])
+
+    },
+
+} + prompts + chunker
+

templates/components/bedrock.jsonnet

@@ -6,6 +6,11 @@ local chunker = import "chunker-recursive.jsonnet";
 
 {
 
+    with:: function(key, value)
+        self + {
+            ["bedrock-" + key]:: value,
+        },
+
     "bedrock-max-output-tokens":: 4096,
     "bedrock-temperature":: 0.0,
     "bedrock-model":: "mistral.mixtral-8x7b-instruct-v0:1",
@@ -15,13 +20,13 @@ local chunker = import "chunker-recursive.jsonnet";
         create:: function(engine)
 
             local envSecrets = engine.envSecrets("bedrock-credentials")
-                .with_env_var("AWS_ID_KEY", "aws-id-key")
+                .with_env_var("AWS_ACCESS_KEY_ID", "aws-id-key")
-                .with_env_var("AWS_SECRET", "aws-secret")
+                .with_env_var("AWS_SECRET_ACCESS_KEY", "aws-secret")
-                .with_env_var("AWS_REGION", "aws-region");
+                .with_env_var("AWS_DEFAULT_REGION", "aws-region");
 
             local container =
                 engine.container("text-completion")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_bedrock)
                     .with_command([
                         "text-completion-bedrock",
                         "-p",
@@ -37,50 +42,18 @@ local chunker = import "chunker-recursive.jsonnet";
                     .with_limits("0.5", "128M")
                     .with_reservations("0.1", "128M");
 
-            local containerRag =
-                engine.container("text-completion-rag")
-                    .with_image(images.trustgraph)
-                    .with_command([
-                        "text-completion-bedrock",
-                        "-p",
-                        url.pulsar,
-                        "-x",
-                        std.toString($["bedrock-max-output-tokens"]),
-                        "-t",
-                        "%0.3f" % $["bedrock-temperature"],
-                        "-m",
-                        $["bedrock-model"],
-                        "-i",
-                        "non-persistent://tg/request/text-completion-rag",
-                        "-o",
-                        "non-persistent://tg/response/text-completion-rag",
-              	    ])
-                    .with_env_var_secrets(envSecrets)
-                    .with_limits("0.5", "128M")
-                    .with_reservations("0.1", "128M");
-
             local containerSet = engine.containers(
                 "text-completion", [ container ]
             );
 
-            local containerSetRag = engine.containers(
-                "text-completion-rag", [ containerRag ]
-            );
-
             local service =
                 engine.internalService(containerSet)
                 .with_port(8000, 8000, "metrics");
 
-            local serviceRag =
-                engine.internalService(containerSetRag)
-                .with_port(8000, 8000, "metrics");
-
             engine.resources([
                 envSecrets,
                 containerSet,
-                containerSetRag,
                 service,
-                serviceRag,
             ])
 
     },

templates/components/cassandra.jsonnet

@@ -12,7 +12,7 @@ cassandra + {
 
             local container =
                 engine.container("store-triples")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "triples-write-cassandra",
                         "-p",
@@ -44,7 +44,7 @@ cassandra + {
 
             local container =
                 engine.container("query-triples")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "triples-query-cassandra",
                         "-p",

templates/components/chunker-recursive.jsonnet

@@ -14,7 +14,7 @@ local prompts = import "prompts/mixtral.jsonnet";
 
             local container =
                 engine.container("chunker")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "chunker-recursive",
                         "-p",

templates/components/claude-rag.jsonnet

@@ -0,0 +1,63 @@
+local base = import "base/base.jsonnet";
+local images = import "values/images.jsonnet";
+local url = import "values/url.jsonnet";
+local prompts = import "prompts/mixtral.jsonnet";
+
+{
+
+    with:: function(key, value)
+        self + {
+            ["claude-rag-" + key]:: value,
+        },
+
+    "claude-rag-model":: "claude-3-sonnet-20240229",
+    "claude-rag-max-output-tokens":: 4096,
+    "claude-rag-temperature":: 0.0,
+
+    "text-completion-rag" +: {
+    
+        create:: function(engine)
+
+            local envSecrets = engine.envSecrets("claude-credentials")
+                .with_env_var("CLAUDE_KEY", "claude-key");
+
+            local containerRag =
+                engine.container("text-completion-rag")
+                    .with_image(images.trustgraph_flow)
+                    .with_command([
+                        "text-completion-claude",
+                        "-p",
+                        url.pulsar,
+                        "-x",
+                        std.toString($["claude-rag-max-output-tokens"]),
+                        "-m",
+                        $["claude-rag-model"],
+                        "-t",
+                        "%0.3f" % $["claude-rag-temperature"],
+                        "-i",
+                        "non-persistent://tg/request/text-completion-rag",
+                        "-o",
+                        "non-persistent://tg/response/text-completion-rag",
+                    ])
+                    .with_env_var_secrets(envSecrets)
+                    .with_limits("0.5", "128M")
+                    .with_reservations("0.1", "128M");
+
+            local containerSetRag = engine.containers(
+                "text-completion-rag", [ containerRag ]
+            );
+
+            local serviceRag =
+                engine.internalService(containerSetRag)
+                .with_port(8000, 8000, "metrics");
+
+            engine.resources([
+                envSecrets,
+                containerSetRag,
+                serviceRag,
+            ])
+
+    },
+
+} + prompts
+

templates/components/claude.jsonnet

@@ -5,6 +5,12 @@ local prompts = import "prompts/mixtral.jsonnet";
 
 {
 
+    with:: function(key, value)
+        self + {
+            ["claude-" + key]:: value,
+        },
+
+    "claude-model":: "claude-3-sonnet-20240229",
     "claude-max-output-tokens":: 4096,
     "claude-temperature":: 0.0,
 
@@ -13,17 +19,19 @@ local prompts = import "prompts/mixtral.jsonnet";
         create:: function(engine)
 
             local envSecrets = engine.envSecrets("claude-credentials")
-                .with_env_var("CLAUDE_KEY_TOKEN", "claude-key");
+                .with_env_var("CLAUDE_KEY", "claude-key");
 
             local container =
                 engine.container("text-completion")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "text-completion-claude",
                         "-p",
                         url.pulsar,
                         "-x",
                         std.toString($["claude-max-output-tokens"]),
+                        "-m",
+                        $["claude-model"],
                         "-t",
                         "%0.3f" % $["claude-temperature"],
                     ])
@@ -31,48 +39,18 @@ local prompts = import "prompts/mixtral.jsonnet";
                     .with_limits("0.5", "128M")
                     .with_reservations("0.1", "128M");
 
-            local containerRag =
-                engine.container("text-completion-rag")
-                    .with_image(images.trustgraph)
-                    .with_command([
-                        "text-completion-claude",
-                        "-p",
-                        url.pulsar,
-                        "-x",
-                        std.toString($["claude-max-output-tokens"]),
-                        "-t",
-                        "%0.3f" % $["claude-temperature"],
-                        "-i",
-                        "non-persistent://tg/request/text-completion-rag",
-                        "-o",
-                        "non-persistent://tg/response/text-completion-rag",
-                    ])
-                    .with_env_var_secrets(envSecrets)
-                    .with_limits("0.5", "128M")
-                    .with_reservations("0.1", "128M");
-
             local containerSet = engine.containers(
                 "text-completion", [ container ]
             );
 
-            local containerSetRag = engine.containers(
-                "text-completion-rag", [ containerRag ]
-            );
-
             local service =
                 engine.internalService(containerSet)
                 .with_port(8000, 8000, "metrics");
 
-            local serviceRag =
-                engine.internalService(containerSetRag)
-                .with_port(8000, 8000, "metrics");
-
             engine.resources([
                 envSecrets,
                 containerSet,
-                containerSetRag,
                 service,
-                serviceRag,
             ])
 
     },

templates/components/cohere-rag.jsonnet

@@ -0,0 +1,56 @@
+local base = import "base/base.jsonnet";
+local images = import "values/images.jsonnet";
+local url = import "values/url.jsonnet";
+local prompts = import "prompts/mixtral.jsonnet";
+
+{
+
+    with:: function(key, value)
+        self + {
+            ["cohere-rag-" + key]:: value,
+        },
+
+    "cohere-rag-temperature":: 0.0,
+
+    "text-completion-rag" +: {
+    
+        create:: function(engine)
+
+            local envSecrets = engine.envSecrets("cohere-credentials")
+                .with_env_var("COHERE_KEY", "cohere-key");
+
+            local containerRag =
+                engine.container("text-completion-rag")
+                    .with_image(images.trustgraph_flow)
+                    .with_command([
+                        "text-completion-cohere",
+                        "-p",
+                        url.pulsar,
+                        "-t",
+                        "%0.3f" % $["cohere-rag-temperature"],
+                        "-i",
+                        "non-persistent://tg/request/text-completion-rag",
+                        "-o",
+                        "non-persistent://tg/response/text-completion-rag",
+                    ])
+                    .with_limits("0.5", "128M")
+                    .with_reservations("0.1", "128M");
+
+            local containerSetRag = engine.containers(
+                "text-completion-rag", [ containerRag ]
+            );
+
+            local serviceRag =
+                engine.internalService(containerSetRag)
+                .with_port(8000, 8000, "metrics");
+
+            engine.resources([
+                envSecrets,
+                containerSetRag,
+                serviceRag,
+            ])
+
+    },
+
+} + prompts
+

templates/components/cohere.jsonnet

@@ -5,9 +5,10 @@ local prompts = import "prompts/mixtral.jsonnet";
 
 {
 
-    // Override chunking
+    with:: function(key, value)
-    "chunk-size":: 150,
+        self + {
-    "chunk-overlap":: 10,
+            ["cohere-" + key]:: value,
+        },
 
     "cohere-temperature":: 0.0,
 
@@ -20,7 +21,7 @@ local prompts = import "prompts/mixtral.jsonnet";
 
             local container =
                 engine.container("text-completion")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "text-completion-cohere",
                         "-p",
@@ -31,45 +32,18 @@ local prompts = import "prompts/mixtral.jsonnet";
                     .with_limits("0.5", "128M")
                     .with_reservations("0.1", "128M");
 
-            local containerRag =
-                engine.container("text-completion-rag")
-                    .with_image(images.trustgraph)
-                    .with_command([
-                        "text-completion-cohere",
-                        "-p",
-                        url.pulsar,
-                        "-t",
-                        "%0.3f" % $["cohere-temperature"],
-                        "-i",
-                        "non-persistent://tg/request/text-completion-rag",
-                        "-o",
-                        "non-persistent://tg/response/text-completion-rag",
-                    ])
-                    .with_limits("0.5", "128M")
-                    .with_reservations("0.1", "128M");
-
             local containerSet = engine.containers(
                 "text-completion", [ container ]
             );
 
-            local containerSetRag = engine.containers(
-                "text-completion-rag", [ containerRag ]
-            );
-
             local service =
                 engine.internalService(containerSet)
                 .with_port(8000, 8000, "metrics");
 
-            local serviceRag =
-                engine.internalService(containerSetRag)
-                .with_port(8000, 8000, "metrics");
-
             engine.resources([
                 envSecrets,
                 containerSet,
-                containerSetRag,
                 service,
-                serviceRag,
             ])
 
     },

templates/components/document-rag.jsonnet

@@ -11,7 +11,7 @@ local prompts = import "prompts/mixtral.jsonnet";
 
             local container =
                 engine.container("document-rag")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "document-rag",
                         "-p",
@@ -39,5 +39,35 @@ local prompts = import "prompts/mixtral.jsonnet";
 
     },
 
+    "document-embeddings" +: {
+    
+        create:: function(engine)
+
+            local container =
+                engine.container("document-embeddings")
+                    .with_image(images.trustgraph_flow)
+                    .with_command([
+                        "document-embeddings",
+                        "-p",
+                        url.pulsar,
+                    ])
+                    .with_limits("1.0", "512M")
+                    .with_reservations("0.5", "512M");
+
+            local containerSet = engine.containers(
+                "document-embeddings", [ container ]
+            );
+
+            local service =
+                engine.internalService(containerSet)
+                .with_port(8000, 8000, "metrics");
+
+            engine.resources([
+                containerSet,
+                service,
+            ])
+
+    },
+
 }
 

templates/components/embeddings-fastembed.jsonnet

@@ -0,0 +1,43 @@
+local base = import "base/base.jsonnet";
+local images = import "values/images.jsonnet";
+local url = import "values/url.jsonnet";
+local prompts = import "prompts/mixtral.jsonnet";
+
+{
+
+    "embeddings-model":: "sentence-transformers/all-MiniLM-L6-v2",
+
+    embeddings +: {
+    
+        create:: function(engine)
+
+            local container =
+                engine.container("embeddings")
+                    .with_image(images.trustgraph_flow)
+                    .with_command([
+                        "embeddings-fastembed",
+                        "-p",
+                        url.pulsar,
+                        "-m",
+                        $["embeddings-model"],
+                    ])
+                    .with_limits("1.0", "400M")
+                    .with_reservations("0.5", "400M");
+
+            local containerSet = engine.containers(
+                "embeddings", [ container ]
+            );
+
+            local service =
+                engine.internalService(containerSet)
+                .with_port(8000, 8000, "metrics");
+
+            engine.resources([
+                containerSet,
+                service,
+            ])
+
+    },
+
+}
+

templates/components/embeddings-hf.jsonnet

@@ -13,7 +13,7 @@ local prompts = import "prompts/mixtral.jsonnet";
 
             local container =
                 engine.container("embeddings")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_hf)
                     .with_command([
                         "embeddings-hf",
                         "-p",

templates/components/embeddings-ollama.jsonnet

@@ -13,7 +13,7 @@ local url = import "values/url.jsonnet";
 
             local container =
                 engine.container("embeddings")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "embeddings-ollama",
                         "-p",

templates/components/falkordb.jsonnet

@@ -13,7 +13,7 @@ falkordb + {
 
             local container =
                 engine.container("store-triples")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "triples-write-falkordb",
                         "-p",
@@ -45,7 +45,7 @@ falkordb + {
 
             local container =
                 engine.container("query-triples")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "triples-query-falkordb",
                         "-p",

templates/components/googleaistudio-rag.jsonnet

@@ -0,0 +1,65 @@
+local base = import "base/base.jsonnet";
+local images = import "values/images.jsonnet";
+local url = import "values/url.jsonnet";
+local prompts = import "prompts/mixtral.jsonnet";
+
+{
+
+    with:: function(key, value)
+        self + {
+            ["googleaistudio-rag-" + key]:: value,
+        },
+
+    "googleaistudio-rag-max-output-tokens":: 4096,
+    "googleaistudio-rag-temperature":: 0.0,
+    "googleaistudio-rag-model":: "gemini-1.5-flash-002",
+
+    "text-completion-rag" +: {
+    
+        create:: function(engine)
+
+            local envSecrets = engine.envSecrets("googleaistudio-credentials")
+                .with_env_var("GOOGLE_AI_STUDIO_KEY", "googleaistudio-key");
+
+            local containerRag =
+                engine.container("text-completion-rag")
+                    .with_image(images.trustgraph_flow)
+                    .with_command([
+                        "text-completion-googleaistudio",
+                        "-p",
+                        url.pulsar,
+                        "-x",
+                        std.toString(
+                            $["googleaistudio-rag-max-output-tokens"]
+                        ),
+                        "-t",
+                        "%0.3f" % $["googleaistudio-rag-temperature"],
+                        "-m",
+                        $["googleaistudio-rag-model"],
+                        "-i",
+                        "non-persistent://tg/request/text-completion-rag",
+                        "-o",
+                        "non-persistent://tg/response/text-completion-rag",
+                    ])
+                    .with_env_var_secrets(envSecrets)
+                    .with_limits("0.5", "128M")
+                    .with_reservations("0.1", "128M");
+
+            local containerSetRag = engine.containers(
+                "text-completion-rag", [ containerRag ]
+            );
+
+            local serviceRag =
+                engine.internalService(containerSetRag)
+                .with_port(8000, 8000, "metrics");
+
+            engine.resources([
+                envSecrets,
+                containerSetRag,
+                serviceRag,
+            ])
+
+    },
+
+} + prompts
+

templates/components/googleaistudio.jsonnet

@@ -5,6 +5,11 @@ local prompts = import "prompts/mixtral.jsonnet";
 
 {
 
+    with:: function(key, value)
+        self + {
+            ["googleaistudio-" + key]:: value,
+        },
+
     "googleaistudio-max-output-tokens":: 4096,
     "googleaistudio-temperature":: 0.0,
     "googleaistudio-model":: "gemini-1.5-flash-002",
@@ -13,12 +18,12 @@ local prompts = import "prompts/mixtral.jsonnet";
     
         create:: function(engine)
 
-            local envSecrets = engine.envSecrets("googleaistudio-key")
+            local envSecrets = engine.envSecrets("googleaistudio-credentials")
                 .with_env_var("GOOGLE_AI_STUDIO_KEY", "googleaistudio-key");
 
             local container =
                 engine.container("text-completion")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "text-completion-googleaistudio",
                         "-p",
@@ -34,50 +39,18 @@ local prompts = import "prompts/mixtral.jsonnet";
                     .with_limits("0.5", "128M")
                     .with_reservations("0.1", "128M");
 
-            local containerRag =
-                engine.container("text-completion-rag")
-                    .with_image(images.trustgraph)
-                    .with_command([
-                        "text-completion-googleaistudio",
-                        "-p",
-                        url.pulsar,
-                        "-x",
-                        std.toString($["googleaistudio-max-output-tokens"]),
-                        "-t",
-                        "%0.3f" % $["googleaistudio-temperature"],
-                        "-m",
-                        $["googleaistudio-model"],
-                        "-i",
-                        "non-persistent://tg/request/text-completion-rag",
-                        "-o",
-                        "non-persistent://tg/response/text-completion-rag",
-                    ])
-                    .with_env_var_secrets(envSecrets)
-                    .with_limits("0.5", "128M")
-                    .with_reservations("0.1", "128M");
-
             local containerSet = engine.containers(
                 "text-completion", [ container ]
             );
 
-            local containerSetRag = engine.containers(
-                "text-completion-rag", [ containerRag ]
-            );
-
             local service =
                 engine.internalService(containerSet)
                 .with_port(8000, 8000, "metrics");
 
-            local serviceRag =
-                engine.internalService(containerSetRag)
-                .with_port(8000, 8000, "metrics");
-
             engine.resources([
                 envSecrets,
                 containerSet,
-                containerSetRag,
                 service,
-                serviceRag,
             ])
 
     },

templates/components/graph-rag.jsonnet

@@ -14,7 +14,7 @@ local url = import "values/url.jsonnet";
 
             local container =
                 engine.container("kg-extract-definitions")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "kg-extract-definitions",
                         "-p",
@@ -44,7 +44,7 @@ local url = import "values/url.jsonnet";
 
             local container =
                 engine.container("kg-extract-relationships")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "kg-extract-relationships",
                         "-p",
@@ -74,7 +74,7 @@ local url = import "values/url.jsonnet";
 
             local container =
                 engine.container("kg-extract-topics")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "kg-extract-topics",
                         "-p",
@@ -104,7 +104,7 @@ local url = import "values/url.jsonnet";
 
             local container =
                 engine.container("graph-rag")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "graph-rag",
                         "-p",
@@ -138,5 +138,35 @@ local url = import "values/url.jsonnet";
 
     },
 
+    "graph-embeddings" +: {
+    
+        create:: function(engine)
+
+            local container =
+                engine.container("graph-embeddings")
+                    .with_image(images.trustgraph_flow)
+                    .with_command([
+                        "graph-embeddings",
+                        "-p",
+                        url.pulsar,
+                    ])
+                    .with_limits("1.0", "512M")
+                    .with_reservations("0.5", "512M");
+
+            local containerSet = engine.containers(
+                "graph-embeddings", [ container ]
+            );
+
+            local service =
+                engine.internalService(containerSet)
+                .with_port(8000, 8000, "metrics");
+
+            engine.resources([
+                containerSet,
+                service,
+            ])
+
+    },
+
 }
 

templates/components/llamafile-rag.jsonnet

@@ -0,0 +1,57 @@
+local base = import "base/base.jsonnet";
+local images = import "values/images.jsonnet";
+local url = import "values/url.jsonnet";
+local prompts = import "prompts/slm.jsonnet";
+
+{
+
+    with:: function(key, value)
+        self + {
+            ["llamafile-rag-" + key]:: value,
+        },
+
+    "llamafile-rag-model":: "LLaMA_CPP",
+
+    "text-completion-rag" +: {
+    
+        create:: function(engine)
+
+            local envSecrets = engine.envSecrets("llamafile-credentials")
+                .with_env_var("LLAMAFILE_URL", "llamafile-url");
+
+            local containerRag =
+                engine.container("text-completion-rag")
+                    .with_image(images.trustgraph_flow)
+                    .with_command([
+                        "text-completion-llamafile",
+                        "-p",
+                        url.pulsar,
+                        "-m",
+                        $["llamafile-rag-model"],
+                        "-i",
+                        "non-persistent://tg/request/text-completion-rag",
+                        "-o",
+                        "non-persistent://tg/response/text-completion-rag",
+                    ])
+                    .with_env_var_secrets(envSecrets)
+                    .with_limits("0.5", "128M")
+                    .with_reservations("0.1", "128M");
+
+            local containerSetRag = engine.containers(
+                "text-completion-rag", [ containerRag ]
+            );
+
+            local serviceRag =
+                engine.internalService(containerSetRag)
+                .with_port(8080, 8080, "metrics");
+
+            engine.resources([
+                envSecrets,
+                containerSetRag,
+                serviceRag,
+            ])
+
+    },
+
+} + prompts
+

templates/components/llamafile.jsonnet

@@ -5,6 +5,11 @@ local prompts = import "prompts/slm.jsonnet";
 
 {
 
+    with:: function(key, value)
+        self + {
+            ["llamafile-" + key]:: value,
+        },
+
     "llamafile-model":: "LLaMA_CPP",
 
     "text-completion" +: {
@@ -16,7 +21,7 @@ local prompts = import "prompts/slm.jsonnet";
 
             local container =
                 engine.container("text-completion")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "text-completion-llamafile",
                         "-p",
@@ -28,46 +33,18 @@ local prompts = import "prompts/slm.jsonnet";
                     .with_limits("0.5", "128M")
                     .with_reservations("0.1", "128M");
 
-            local containerRag =
-                engine.container("text-completion-rag")
-                    .with_image(images.trustgraph)
-                    .with_command([
-                        "text-completion-llamafile",
-                        "-p",
-                        url.pulsar,
-                        "-m",
-                        $["llamafile-model"],
-                        "-i",
-                        "non-persistent://tg/request/text-completion-rag",
-                        "-o",
-                        "non-persistent://tg/response/text-completion-rag",
-                    ])
-                    .with_env_var_secrets(envSecrets)
-                    .with_limits("0.5", "128M")
-                    .with_reservations("0.1", "128M");
-
             local containerSet = engine.containers(
                 "text-completion", [ container ]
             );
 
-            local containerSetRag = engine.containers(
-                "text-completion-rag", [ containerRag ]
-            );
-
             local service =
                 engine.internalService(containerSet)
                 .with_port(8080, 8080, "metrics");
 
-            local serviceRag =
-                engine.internalService(containerSetRag)
-                .with_port(8080, 8080, "metrics");
-
             engine.resources([
                 envSecrets,
                 containerSet,
-                containerSetRag,
                 service,
-                serviceRag,
             ])
 
     },

templates/components/memgraph.jsonnet

@@ -14,7 +14,7 @@ memgraph + {
 
             local container =
                 engine.container("store-triples")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "triples-write-memgraph",
                         "-p",
@@ -48,7 +48,7 @@ memgraph + {
 
             local container =
                 engine.container("query-triples")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "triples-query-memgraph",
                         "-p",

templates/components/milvus.jsonnet

@@ -12,7 +12,7 @@ milvus + {
 
             local container =
                 engine.container("store-graph-embeddings")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "ge-write-milvus",
                         "-p",
@@ -44,7 +44,7 @@ milvus + {
 
             local container =
                 engine.container("query-graph-embeddings")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "ge-query-milvus",
                         "-p",
@@ -76,7 +76,7 @@ milvus + {
 
             local container =
                 engine.container("store-doc-embeddings")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "de-write-milvus",
                         "-p",
@@ -108,7 +108,7 @@ milvus + {
 
             local container =
                 engine.container("query-doc-embeddings")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "de-query-milvus",
                         "-p",

templates/components/neo4j.jsonnet

@@ -13,7 +13,7 @@ neo4j + {
 
             local container =
                 engine.container("store-triples")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "triples-write-neo4j",
                         "-p",
@@ -45,7 +45,7 @@ neo4j + {
 
             local container =
                 engine.container("query-triples")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "triples-query-neo4j",
                         "-p",

templates/components/ollama-rag.jsonnet

@@ -0,0 +1,57 @@
+local base = import "base/base.jsonnet";
+local images = import "values/images.jsonnet";
+local url = import "values/url.jsonnet";
+local prompts = import "prompts/mixtral.jsonnet";
+
+{
+
+    with:: function(key, value)
+        self + {
+            ["ollama-rag-" + key]:: value,
+        },
+
+    "ollama-rag-model":: "gemma2:9b",
+
+    "text-completion-rag" +: {
+    
+        create:: function(engine)
+
+            local envSecrets = engine.envSecrets("ollama-credentials")
+                .with_env_var("OLLAMA_HOST", "ollama-host");
+
+            local containerRag =
+                engine.container("text-completion-rag")
+                    .with_image(images.trustgraph_flow)
+                    .with_command([
+                        "text-completion-ollama",
+                        "-p",
+                        url.pulsar,
+                        "-m",
+                        $["ollama-rag-model"],
+                        "-i",
+                        "non-persistent://tg/request/text-completion-rag",
+                        "-o",
+                        "non-persistent://tg/response/text-completion-rag",
+                    ])
+                    .with_env_var_secrets(envSecrets)
+                    .with_limits("0.5", "128M")
+                    .with_reservations("0.1", "128M");
+
+            local containerSetRag = engine.containers(
+                "text-completion-rag", [ containerRag ]
+            );
+
+            local serviceRag =
+                engine.internalService(containerSetRag)
+                .with_port(8080, 8080, "metrics");
+
+            engine.resources([
+                envSecrets,
+                containerSetRag,
+                serviceRag,
+            ])
+
+    },
+
+} + prompts
+

templates/components/ollama.jsonnet

@@ -5,6 +5,11 @@ local prompts = import "prompts/mixtral.jsonnet";
 
 {
 
+    with:: function(key, value)
+        self + {
+            ["ollama-" + key]:: value,
+        },
+
     "ollama-model":: "gemma2:9b",
 
     "text-completion" +: {
@@ -16,7 +21,7 @@ local prompts = import "prompts/mixtral.jsonnet";
 
             local container =
                 engine.container("text-completion")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "text-completion-ollama",
                         "-p",
@@ -28,46 +33,18 @@ local prompts = import "prompts/mixtral.jsonnet";
                     .with_limits("0.5", "128M")
                     .with_reservations("0.1", "128M");
 
-            local containerRag =
-                engine.container("text-completion-rag")
-                    .with_image(images.trustgraph)
-                    .with_command([
-                        "text-completion-ollama",
-                        "-p",
-                        url.pulsar,
-                        "-m",
-                        $["ollama-model"],
-                        "-i",
-                        "non-persistent://tg/request/text-completion-rag",
-                        "-o",
-                        "non-persistent://tg/response/text-completion-rag",
-                    ])
-                    .with_env_var_secrets(envSecrets)
-                    .with_limits("0.5", "128M")
-                    .with_reservations("0.1", "128M");
-
             local containerSet = engine.containers(
                 "text-completion", [ container ]
             );
 
-            local containerSetRag = engine.containers(
-                "text-completion-rag", [ containerRag ]
-            );
-
             local service =
                 engine.internalService(containerSet)
                 .with_port(8080, 8080, "metrics");
 
-            local serviceRag =
-                engine.internalService(containerSetRag)
-                .with_port(8080, 8080, "metrics");
-
             engine.resources([
                 envSecrets,
                 containerSet,
-                containerSetRag,
                 service,
-                serviceRag,
             ])
 
     },

templates/components/openai-rag.jsonnet

@@ -0,0 +1,63 @@
+local base = import "base/base.jsonnet";
+local images = import "values/images.jsonnet";
+local url = import "values/url.jsonnet";
+local prompts = import "prompts/mixtral.jsonnet";
+
+{
+
+    with:: function(key, value)
+        self + {
+            ["openai-rag-" + key]:: value,
+        },
+
+    "openai-rag-max-output-tokens":: 4096,
+    "openai-rag-temperature":: 0.0,
+    "openai-rag-model":: "GPT-3.5-Turbo",
+
+    "text-completion-rag" +: {
+    
+        create:: function(engine)
+
+            local envSecrets = engine.envSecrets("openai-credentials")
+                .with_env_var("OPENAI_TOKEN", "openai-token");
+
+            local containerRag =
+                engine.container("text-completion-rag")
+                    .with_image(images.trustgraph_flow)
+                    .with_command([
+                        "text-completion-openai",
+                        "-p",
+                        url.pulsar,
+                        "-x",
+                        std.toString($["openai-rag-max-output-tokens"]),
+                        "-t",
+                        "%0.3f" % $["openai-rag-temperature"],
+                        "-m",
+                        $["openai-rag-model"],
+                        "-i",
+                        "non-persistent://tg/request/text-completion-rag",
+                        "-o",
+                        "non-persistent://tg/response/text-completion-rag",
+                    ])
+                    .with_env_var_secrets(envSecrets)
+                    .with_limits("0.5", "128M")
+                    .with_reservations("0.1", "128M");
+
+            local containerSetRag = engine.containers(
+                "text-completion-rag", [ containerRag ]
+            );
+
+            local serviceRag =
+                engine.internalService(containerSetRag)
+                .with_port(8080, 8080, "metrics");
+
+            engine.resources([
+                envSecrets,
+                containerSetRag,
+                serviceRag,
+            ])
+
+    },
+
+} + prompts
+

templates/components/openai.jsonnet

@@ -5,6 +5,11 @@ local prompts = import "prompts/mixtral.jsonnet";
 
 {
 
+    with:: function(key, value)
+        self + {
+            ["openai-" + key]:: value,
+        },
+
     "openai-max-output-tokens":: 4096,
     "openai-temperature":: 0.0,
     "openai-model":: "GPT-3.5-Turbo",
@@ -18,7 +23,7 @@ local prompts = import "prompts/mixtral.jsonnet";
 
             local container =
                 engine.container("text-completion")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "text-completion-openai",
                         "-p",
@@ -34,50 +39,18 @@ local prompts = import "prompts/mixtral.jsonnet";
                     .with_limits("0.5", "128M")
                     .with_reservations("0.1", "128M");
 
-            local containerRag =
-                engine.container("text-completion-rag")
-                    .with_image(images.trustgraph)
-                    .with_command([
-                        "text-completion-openai",
-                        "-p",
-                        url.pulsar,
-                        "-x",
-                        std.toString($["openai-max-output-tokens"]),
-                        "-t",
-                        "%0.3f" % $["openai-temperature"],
-                        "-m",
-                        $["openai-model"],
-                        "-i",
-                        "non-persistent://tg/request/text-completion-rag",
-                        "-o",
-                        "non-persistent://tg/response/text-completion-rag",
-                    ])
-                    .with_env_var_secrets(envSecrets)
-                    .with_limits("0.5", "128M")
-                    .with_reservations("0.1", "128M");
-
             local containerSet = engine.containers(
                 "text-completion", [ container ]
             );
 
-            local containerSetRag = engine.containers(
-                "text-completion-rag", [ containerRag ]
-            );
-
             local service =
                 engine.internalService(containerSet)
                 .with_port(8080, 8080, "metrics");
 
-            local serviceRag =
-                engine.internalService(containerSetRag)
-                .with_port(8080, 8080, "metrics");
-
             engine.resources([
                 envSecrets,
                 containerSet,
-                containerSetRag,
                 service,
-                serviceRag,
             ])
 
     },

templates/components/pinecone.jsonnet

@@ -17,7 +17,7 @@ local cassandra_hosts = "cassandra";
 
             local container =
                 engine.container("store-graph-embeddings")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "ge-write-pinecone",
                         "-p",
@@ -52,7 +52,7 @@ local cassandra_hosts = "cassandra";
 
             local container =
                 engine.container("query-graph-embeddings")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "ge-query-pinecone",
                         "-p",
@@ -87,7 +87,7 @@ local cassandra_hosts = "cassandra";
 
             local container =
                 engine.container("store-doc-embeddings")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "de-write-pinecone",
                         "-p",
@@ -122,7 +122,7 @@ local cassandra_hosts = "cassandra";
 
             local container =
                 engine.container("query-doc-embeddings")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "de-query-pinecone",
                         "-p",

templates/components/prompt-overrides.jsonnet

@@ -1,7 +1,3 @@
-local base = import "base/base.jsonnet";
-local images = import "values/images.jsonnet";
-local url = import "values/url.jsonnet";
-local prompts = import "prompts/mixtral.jsonnet";
 local default_prompts = import "prompts/default-prompts.jsonnet";
 
 {

templates/components/prompt-template.jsonnet

@@ -44,7 +44,7 @@ local default_prompts = import "prompts/default-prompts.jsonnet";
 
             local container =
                 engine.container("prompt")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "prompt-template",
                         "-p",
@@ -84,7 +84,7 @@ local default_prompts = import "prompts/default-prompts.jsonnet";
 
             local container =
                 engine.container("prompt-rag")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "prompt-template",
                         "-p",

templates/components/pulsar.jsonnet

@@ -2,32 +2,114 @@ local base = import "base/base.jsonnet";
 local images = import "values/images.jsonnet";
 local url = import "values/url.jsonnet";
 
+// This is a Pulsar configuration.  Non-standalone mode so we deploy
+// individual components: bookkeeper, broker and zookeeper.
+// 
+// This also deploys the TrustGraph 'admin' container which initialises
+// TrustGraph-specific namespaces etc.
+
 {
 
     "pulsar" +: {
 
         create:: function(engine)
 
-//            local confVolume = engine.volume("pulsar-conf").with_size("2G");
+            // Zookeeper volume
-            local dataVolume = engine.volume("pulsar-data").with_size("20G");
+            local zkVolume = engine.volume("zookeeper").with_size("1G");
 
-            local container =
+            // Zookeeper container
+            local zkContainer = 
+                engine.container("zookeeper")
+                    .with_image(images.pulsar)
+                    .with_command([
+                        "bash",
+                        "-c",
+                        "bin/apply-config-from-env.py conf/zookeeper.conf && bin/generate-zookeeper-config.sh conf/zookeeper.conf && exec bin/pulsar zookeeper"
+                    ])
+                    .with_limits("1", "400M")
+                    .with_reservations("0.05", "400M")
+                    .with_user("0:1000")
+                    .with_volume_mount(zkVolume, "/pulsar/data/zookeeper")
+                    .with_environment({
+                        "metadataStoreUrl": "zk:zookeeper:2181",
+                        "PULSAR_MEM": "-Xms256m -Xmx256m -XX:MaxDirectMemorySize=256m",
+                    })
+                    .with_port(2181, 2181, "zookeeper")
+                    .with_port(2888, 2888, "zookeeper2")
+                    .with_port(3888, 3888, "zookeeper3");
+
+            // Pulsar cluster init container
+            local initContainer =
+                engine.container("pulsar-init")
+                    .with_image(images.pulsar)
+                    .with_command([
+                        "bash",
+                        "-c",
+                        "sleep 10 && bin/pulsar initialize-cluster-metadata --cluster cluster-a --zookeeper zookeeper:2181 --configuration-store zookeeper:2181 --web-service-url http://pulsar:8080 --broker-service-url pulsar://pulsar:6650",
+                    ])
+                    .with_limits("1", "512M")
+                    .with_reservations("0.05", "512M")
+                    .with_environment({
+                        "PULSAR_MEM": "-Xms256m -Xmx256m -XX:MaxDirectMemorySize=256m",
+                    });
+
+
+            // Bookkeeper volume
+            local bookieVolume = engine.volume("bookie").with_size("20G");
+
+            // Bookkeeper container
+            local bookieContainer = 
+                engine.container("bookie")
+                    .with_image(images.pulsar)
+                    .with_command([
+                        "bash",
+                        "-c",
+                        "bin/apply-config-from-env.py conf/bookkeeper.conf && exec bin/pulsar bookie"
+                        // false ^ causes this to be a 'failure' exit.
+                    ])
+                    .with_limits("1", "800M")
+                    .with_reservations("0.1", "800M")
+                    .with_user("0:1000")
+                    .with_volume_mount(bookieVolume, "/pulsar/data/bookkeeper")
+                    .with_environment({
+                        "clusterName": "cluster-a",
+                        "zkServers": "zookeeper:2181",
+                        "bookieId": "bookie",
+                        "metadataStoreUri": "metadata-store:zk:zookeeper:2181",
+                        "advertisedAddress": "bookie",
+                        "BOOKIE_MEM": "-Xms512m -Xmx512m -XX:MaxDirectMemorySize=256m",
+                    })
+                    .with_port(3181, 3181, "bookie");
+
+            // Pulsar broker, stateless (uses ZK and Bookkeeper for state)
+            local brokerContainer = 
                 engine.container("pulsar")
                     .with_image(images.pulsar)
-                    .with_command(["bin/pulsar", "standalone"])
+                    .with_command([
+                        "bash",
+                        "-c",
+                        "bin/apply-config-from-env.py conf/broker.conf && exec bin/pulsar broker"
+                    ])
+                    .with_limits("1", "800M")
+                    .with_reservations("0.1", "800M")
                     .with_environment({
-                        "PULSAR_MEM": "-Xms600M -Xmx600M"
+                        "metadataStoreUrl": "zk:zookeeper:2181",
+                        "zookeeperServers": "zookeeper:2181",
+                        "clusterName": "cluster-a",
+                        "managedLedgerDefaultEnsembleSize": "1",
+                        "managedLedgerDefaultWriteQuorum": "1",
+                        "managedLedgerDefaultAckQuorum": "1",
+                        "advertisedAddress": "pulsar",
+                        "advertisedListeners": "external:pulsar://pulsar:6650,localhost:pulsar://localhost:6650",
+                        "PULSAR_MEM": "-Xms512m -Xmx512m -XX:MaxDirectMemorySize=256m",
                     })
-                    .with_limits("2.0", "1500M")
+                    .with_port(6650, 6650, "pulsar")
-                    .with_reservations("1.0", "1500M")
+                    .with_port(8080, 8080, "admin");
-//                    .with_volume_mount(confVolume, "/pulsar/conf")
-                    .with_volume_mount(dataVolume, "/pulsar/data")
-                    .with_port(6650, 6650, "bookie")
-                    .with_port(8080, 8080, "http");
 
+            // Trustgraph Pulsar initialisation
             local adminContainer =
-                engine.container("init-pulsar")
+                engine.container("init-trustgraph")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "tg-init-pulsar",
                         "-p",
@@ -36,10 +118,32 @@ local url = import "values/url.jsonnet";
                     .with_limits("1", "128M")
                     .with_reservations("0.1", "128M");
 
-            local containerSet = engine.containers(
+            // Container sets
+            local zkContainerSet = engine.containers(
+                "zookeeper",
+                [
+                    zkContainer,
+                ]
+            );
+
+            local initContainerSet = engine.containers(
+                "init-pulsar",
+                [
+                    initContainer,
+                ]
+            );
+
+            local bookieContainerSet = engine.containers(
+                "bookie",
+                [
+                    bookieContainer,
+                ]
+            );
+
+            local brokerContainerSet = engine.containers(
                 "pulsar",
                 [
-                    container
+                    brokerContainer,
                 ]
             );
 
@@ -50,17 +154,35 @@ local url = import "values/url.jsonnet";
                 ]
             );
 
-            local service =
+            // Zookeeper service
-                engine.service(containerSet)
+            local zkService =
-                .with_port(6650, 6650, "bookie")
+                engine.service(zkContainerSet)
-                .with_port(8080, 8080, "http");
+                .with_port(2181, 2181, "zookeeper")
+                .with_port(2888, 2888, "zookeeper2")
+                .with_port(3888, 3888, "zookeeper3");
+
+            // Bookkeeper service
+            local bookieService =
+                engine.service(bookieContainerSet)
+                .with_port(3181, 3181, "bookie");
+
+            // Pulsar broker service
+            local brokerService =
+                engine.service(brokerContainerSet)
+                .with_port(6650, 6650, "pulsar")
+                .with_port(8080, 8080, "admin");
 
             engine.resources([
-//                confVolume,
+                zkVolume,
-                dataVolume,
+                bookieVolume,
-                containerSet,
+                zkContainerSet,
+                initContainerSet,
+                bookieContainerSet,
+                brokerContainerSet,
                 adminContainerSet,
-                service,
+                zkService,
+                bookieService,
+                brokerService,
             ])
 
     }

templates/components/qdrant.jsonnet

@@ -12,7 +12,7 @@ qdrant + {
 
             local container =
                 engine.container("store-graph-embeddings")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "ge-write-qdrant",
                         "-p",
@@ -44,7 +44,7 @@ qdrant + {
 
             local container =
                 engine.container("query-graph-embeddings")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "ge-query-qdrant",
                         "-p",
@@ -76,7 +76,7 @@ qdrant + {
 
             local container =
                 engine.container("store-doc-embeddings")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "de-write-qdrant",
                         "-p",
@@ -108,7 +108,7 @@ qdrant + {
 
             local container =
                 engine.container("query-doc-embeddings")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "de-query-qdrant",
                         "-p",

templates/components/trustgraph.jsonnet

@@ -1,7 +1,6 @@
 local base = import "base/base.jsonnet";
 local images = import "values/images.jsonnet";
 local url = import "values/url.jsonnet";
-local prompt = import "prompt-template.jsonnet";
 
 {
 
@@ -22,7 +21,7 @@ local prompt = import "prompt-template.jsonnet";
 
             local container =
                 engine.container("api-gateway")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "api-gateway",
                         "-p",
@@ -61,7 +60,7 @@ local prompt = import "prompt-template.jsonnet";
 
             local container =
                 engine.container("chunker")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "chunker-token",
                         "-p",
@@ -95,7 +94,7 @@ local prompt = import "prompt-template.jsonnet";
 
             local container =
                 engine.container("pdf-decoder")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "pdf-decoder",
                         "-p",
@@ -119,43 +118,13 @@ local prompt = import "prompt-template.jsonnet";
 
     },
 
-    "vectorize" +: {
-    
-        create:: function(engine)
-
-            local container =
-                engine.container("vectorize")
-                    .with_image(images.trustgraph)
-                    .with_command([
-                        "embeddings-vectorize",
-                        "-p",
-                        url.pulsar,
-                    ])
-                    .with_limits("1.0", "512M")
-                    .with_reservations("0.5", "512M");
-
-            local containerSet = engine.containers(
-                "vectorize", [ container ]
-            );
-
-            local service =
-                engine.internalService(containerSet)
-                .with_port(8000, 8000, "metrics");
-
-            engine.resources([
-                containerSet,
-                service,
-            ])
-
-    },
-
     "metering" +: {
     
         create:: function(engine)
 
             local container =
                 engine.container("metering")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "metering",
                         "-p",
@@ -185,7 +154,7 @@ local prompt = import "prompt-template.jsonnet";
 
             local container =
                 engine.container("metering-rag")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "metering",
                         "-p",
@@ -211,5 +180,5 @@ local prompt = import "prompt-template.jsonnet";
 
     },
 
-} + prompt
+}
 

templates/components/vertexai-rag.jsonnet

@@ -0,0 +1,74 @@
+local base = import "base/base.jsonnet";
+local images = import "values/images.jsonnet";
+local url = import "values/url.jsonnet";
+local prompts = import "prompts/mixtral.jsonnet";
+
+{
+
+    with:: function(key, value)
+        self + {
+            ["vertexai-rag-" + key]:: value,
+        },
+
+    "vertexai-rag-model":: "gemini-1.0-pro-001",
+    "vertexai-rag-private-key":: "/vertexai/private.json",
+    "vertexai-rag-region":: "us-central1",
+    "vertexai-rag-max-output-tokens":: 4096,
+    "vertexai-rag-temperature":: 0.0,
+
+    "text-completion-rag" +: {
+    
+        create:: function(engine)
+
+            local cfgVol = engine.secretVolume(
+	        "vertexai-creds",
+	        "./vertexai",
+		{
+		    "private.json": importstr "vertexai/private.json",
+		}
+            );
+
+            local container =
+                engine.container("text-completion-rag")
+                    .with_image(images.trustgraph_vertexai)
+                    .with_command([
+                        "text-completion-vertexai",
+                        "-p",
+                        url.pulsar,
+                        "-k",
+                        $["vertexai-rag-private-key"],
+                        "-r",
+                        $["vertexai-rag-region"],
+                        "-x",
+                        std.toString($["vertexai-rag-max-output-tokens"]),
+                        "-t",
+                        "%0.3f" % $["vertexai-rag-temperature"],
+                        "-m",
+                        $["vertexai-rag-model"],
+                        "-i",
+                        "non-persistent://tg/request/text-completion-rag",
+                        "-o",
+                        "non-persistent://tg/response/text-completion-rag",
+                    ])
+                    .with_limits("0.5", "256M")
+                    .with_reservations("0.1", "256M")
+                    .with_volume_mount(cfgVol, "/vertexai");
+
+            local containerSet = engine.containers(
+                "text-completion-rag", [ container ]
+            );
+
+            local service =
+                engine.internalService(containerSet)
+                .with_port(8000, 8000, "metrics");
+
+            engine.resources([
+                cfgVol,
+                containerSet,
+                service,
+            ])
+
+    }
+
+} + prompts
+

templates/components/vertexai.jsonnet

@@ -5,6 +5,11 @@ local prompts = import "prompts/mixtral.jsonnet";
 
 {
 
+    with:: function(key, value)
+        self + {
+            ["vertexai-" + key]:: value,
+        },
+
     "vertexai-model":: "gemini-1.0-pro-001",
     "vertexai-private-key":: "/vertexai/private.json",
     "vertexai-region":: "us-central1",
@@ -25,7 +30,7 @@ local prompts = import "prompts/mixtral.jsonnet";
 
             local container =
                 engine.container("text-completion")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_vertexai)
                     .with_command([
                         "text-completion-vertexai",
                         "-p",
@@ -61,59 +66,5 @@ local prompts = import "prompts/mixtral.jsonnet";
 
     },
 
-    "text-completion-rag" +: {
-    
-        create:: function(engine)
-
-            local cfgVol = engine.secretVolume(
-	        "vertexai-creds",
-	        "./vertexai",
-		{
-		    "private.json": importstr "vertexai/private.json",
-		}
-            );
-
-            local container =
-                engine.container("text-completion-rag")
-                    .with_image(images.trustgraph)
-                    .with_command([
-                        "text-completion-vertexai",
-                        "-p",
-                        url.pulsar,
-                        "-k",
-                        $["vertexai-private-key"],
-                        "-r",
-                        $["vertexai-region"],
-                        "-x",
-                        std.toString($["vertexai-max-output-tokens"]),
-                        "-t",
-                        "%0.3f" % $["vertexai-temperature"],
-                        "-m",
-                        $["vertexai-model"],
-                        "-i",
-                        "non-persistent://tg/request/text-completion-rag",
-                        "-o",
-                        "non-persistent://tg/response/text-completion-rag",
-                    ])
-                    .with_limits("0.5", "256M")
-                    .with_reservations("0.1", "256M")
-                    .with_volume_mount(cfgVol, "/vertexai");
-
-            local containerSet = engine.containers(
-                "text-completion-rag", [ container ]
-            );
-
-            local service =
-                engine.internalService(containerSet)
-                .with_port(8000, 8000, "metrics");
-
-            engine.resources([
-                cfgVol,
-                containerSet,
-                service,
-            ])
-
-    }
-
 } + prompts
 

templates/engine/docker-compose.jsonnet

@@ -22,6 +22,8 @@
 
         with_image:: function(x) self + { image: x },
 
+        with_user:: function(x) self + { user: x },
+
         with_command:: function(x) self + { command: x },
 
         with_environment:: function(x) self + {
@@ -75,6 +77,10 @@
                 { command: container.command }
                 else {}) +
 
+                (if std.objectHas(container, "user") then
+                { user: container.user }
+                else {}) +
+
                 (if ! std.isEmpty(container.environment) then
                 { environment: container.environment }
                 else {}) +

templates/engine/k8s.jsonnet

@@ -14,6 +14,8 @@
 
         with_image:: function(x) self + { image: x },
 
+        with_user:: function(x) self + { user: x },
+
         with_command:: function(x) self + { command: x },
 
         with_environment:: function(x) self + {

templates/generate

@@ -7,6 +7,8 @@ import logging
 import os
 import sys
 import zipfile
+import pathlib
+from io import BytesIO
 
 logger = logging.getLogger("generate")
 logging.basicConfig(level=logging.INFO, format='%(message)s')
@@ -15,9 +17,13 @@ private_json = "Put your GCP private.json here"
 
 class Generator:
 
-    def __init__(self, config, base="./templates/", version="0.0.0"):
+    def __init__(
+        self, config, templates="./templates/", resources="./resources",
+        version="0.0.0",
+    ):
 
-        self.jsonnet_base = base
+        self.templates = pathlib.Path(templates)
+        self.resources = pathlib.Path(resources)
         self.config = config
         self.version = f"\"{version}\"".encode("utf-8")
 
@@ -34,25 +40,27 @@ class Generator:
             path = os.path.join(".", dir, filename)
             return str(path), self.config
 
-        if filename == "version.jsonnet" and dir == "./templates/values/":
+        if filename == "version.jsonnet" and dir == "templates/values/":
             path = os.path.join(".", dir, filename)
             return str(path), self.version
 
         if dir:
             candidates = [
-                os.path.join(".", dir, filename),
+                self.templates.joinpath(dir, filename),
-                os.path.join(".", filename)
+                self.templates.joinpath(filename),
+                self.resources.joinpath(dir, filename),
+                self.resources.joinpath(filename),
             ]
         else:
             candidates = [
-                os.path.join(".", filename)
+                self.templates.joinpath(filename),
             ]
 
         try:
 
             if filename == "vertexai/private.json":
 
-                return candidates[0], private_json.encode("utf-8")
+                return str(candidates[0]), private_json.encode("utf-8")
 
             for c in candidates:
                 logger.debug("Try: %s", c)
@@ -68,73 +76,157 @@ class Generator:
                 
         except:
 
-            path = os.path.join(self.jsonnet_base, filename)
+            path = os.path.join(self.templates, filename)
             logger.debug("Try: %s", path)
             with open(path, "rb") as f:
                 logger.debug("Loaded: %s", path)
                 return str(path), f.read()
 
+class Packager:
+
+    def __init__(self):
+        self.templates = pathlib.Path("./templates")
+        self.resources = pathlib.Path("./resources")
+
+    def process(
+            self, config, version="0.0.0", platform="docker-compose",
+    ):
+
+        config = config.encode("utf-8")
+
+        gen = Generator(
+            config, templates=self.templates, resources=self.resources,
+            version=version
+        )
+
+        path = self.templates.joinpath(
+            f"config-to-{platform}.jsonnet"
+        )
+        wrapper = path.read_text()
+
+        processed = gen.process(wrapper)
+
+        return processed
+    
+    def generate(self, config, version, platform):
+
+        logger.info(f"Generating for platform={platform} version={version}")
+
+        try:
+
+            if platform in set(["docker-compose", "podman-compose"]):
+                return self.generate_docker_compose(
+                    "docker-compose", version, config
+                )
+            elif platform in set(["minikube-k8s", "gcp-k8s"]):
+                return self.generate_k8s(
+                    platform, version, config
+                )
+            else:
+                raise RuntimeError("Bad configuration")
+
+        except Exception as e:
+            logging.error(f"Exception: {e}")
+            raise e
+
+    def generate_docker_compose(self, platform, version, config):
+
+        processed = self.process(
+            config, platform=platform, version=version
+        )
+
+        y = yaml.dump(processed)
+
+        mem = BytesIO()
+
+        with zipfile.ZipFile(mem, mode='w') as out:
+
+            def output(name, content):
+                logger.info(f"Adding {name}...")
+                out.writestr(name, content)
+
+            fname = "docker-compose.yaml"
+
+            output(fname, y)
+
+            # Grafana config
+            path = self.resources.joinpath(
+                "grafana/dashboards/dashboard.json"
+            )
+            res = path.read_text()
+            output("grafana/dashboards/dashboard.json", res)
+
+            path = self.resources.joinpath(
+                "grafana/provisioning/dashboard.yml"
+            )
+            res = path.read_text()
+            output("grafana/provisioning/dashboard.yml", res)
+
+            path = self.resources.joinpath(
+                "grafana/provisioning/datasource.yml"
+            )
+            res = path.read_text()
+            output("grafana/provisioning/datasource.yml", res)
+
+            # Prometheus config
+            path = self.resources.joinpath(
+                "prometheus/prometheus.yml"
+            )
+            res = path.read_text()
+            output("prometheus/prometheus.yml", res)
+
+        logger.info("Generation complete.")
+
+        return mem.getvalue()
+
+    def generate_k8s(self, platform, version, config):
+
+        processed = self.process(
+            config, platform=platform, version=version
+        )
+
+        y = yaml.dump(processed)
+
+        mem = BytesIO()
+
+        with zipfile.ZipFile(mem, mode='w') as out:
+
+            def output(name, content):
+                logger.info(f"Adding {name}...")
+                out.writestr(name, content)
+
+            fname = "resources.yaml"
+
+            output(fname, y)
+
+        logger.info("Generation complete.")
+
+        return mem.getvalue()
+            
 def main():
 
-    if len(sys.argv) != 3:
+    if len(sys.argv) != 4:
         print()
         print("Usage:")
-        print("  generate <outfile> <version> < input.json")
+        print("  generate <outfile> <version> <platform> < input.json")
         print()
         sys.exit(1)
 
     outfile = sys.argv[1]
     version = sys.argv[2]
+    platform = sys.argv[3]
 
     cfg = sys.stdin.read()
-    cfg = json.loads(cfg)
     
     logger.info(f"Outputting to {outfile}...")
 
-    with zipfile.ZipFile(outfile, mode='w') as out:
+    p = Packager()
+    resp = p.generate(cfg, version, platform)
 
-        def output(name, content):
+    with open(outfile, "wb") as f:
-            logger.info(f"Adding {name}...")
+        f.write(resp)
-            out.writestr(name, content)
 
-        fname = "tg-launch.yaml"
+    return
-
-        platform = "docker-compose"
-
-        with open(f"./templates/config-to-{platform}.jsonnet", "r") as f:
-            wrapper = f.read()
-
-        gen = Generator(json.dumps(cfg).encode("utf-8"), version=version)
-
-        processed = gen.process(wrapper)
-
-        y = yaml.dump(processed)
-
-        output(fname, y)
-
-        # Placeholder for the private.json file.  Won't put actual credentials
-        # here.
-        output("docker-compose/vertexai/private.json", private_json)
-
-        # Grafana config
-        with open("grafana/dashboards/dashboard.json") as f:
-            output(
-                "docker-compose/grafana/dashboards/dashboard.json", f.read()
-            )
-
-        with open("grafana/provisioning/dashboard.yml") as f:
-            output(
-                "docker-compose/grafana/provisioning/dashboard.yml", f.read()
-            )
-
-        with open("grafana/provisioning/datasource.yml") as f:
-            output(
-                "docker-compose/grafana/provisioning/datasource.yml", f.read()
-            )
-
-        # Prometheus config
-        with open("prometheus/prometheus.yml") as f:
-            output("docker-compose/prometheus/prometheus.yml", f.read())
 
 main()
 

templates/generate-all

@@ -87,8 +87,18 @@ def full_config_object(
 ):
 
     return config_object([
-        graph_store, "pulsar", vector_store, embeddings,
+        "triple-store-" + graph_store,
-        "graph-rag", "grafana", "trustgraph", llm, "workbench-ui",
+        "pulsar",
+        "vector-store-" + vector_store,
+        embeddings,
+        "graph-rag",
+        "grafana",
+        "trustgraph-base",
+        llm,
+        llm + "-rag",
+        "workbench-ui",
+        "prompt-template",
+        "agent-manager-react",
     ])
 
 def generate_config(

templates/stores/cassandra.jsonnet

@@ -13,7 +13,7 @@ local images = import "values/images.jsonnet";
                 engine.container("cassandra")
                     .with_image(images.cassandra)
                     .with_environment({
-                        JVM_OPTS: "-Xms300M -Xmx300M",
+                        JVM_OPTS: "-Xms300M -Xmx300M -Dcassandra.skip_wait_for_gossip_to_settle=0",
                     })
                     .with_limits("1.0", "1000M")
                     .with_reservations("0.5", "1000M")

templates/stores/memgraph.jsonnet

@@ -7,6 +7,8 @@ local images = import "values/images.jsonnet";
     
         create:: function(engine)
 
+            local vol = engine.volume("memgraph").with_size("20G");
+
             local container =
                 engine.container("memgraph")
                     .with_image(images.memgraph_mage)
@@ -16,7 +18,8 @@ local images = import "values/images.jsonnet";
                     .with_limits("1.0", "1000M")
                     .with_reservations("0.5", "1000M")
                     .with_port(7474, 7474, "api")
-                    .with_port(7687, 7687, "api2");
+                    .with_port(7687, 7687, "api2")
+                    .with_volume_mount(vol, "/var/lib/memgraph");
 
             local containerSet = engine.containers(
                 "memgraph", [ container ]
@@ -28,6 +31,7 @@ local images = import "values/images.jsonnet";
                 .with_port(7687, 7687, "api2");
 
             engine.resources([
+                vol,
                 containerSet,
                 service,
             ])
@@ -65,4 +69,3 @@ local images = import "values/images.jsonnet";
     },
 
 }
-

templates/stores/neo4j.jsonnet

@@ -14,12 +14,14 @@ local images = import "values/images.jsonnet";
                     .with_image(images.neo4j)
                     .with_environment({
                         NEO4J_AUTH: "neo4j/password",
+                        NEO4J_server_memory_pagecache_size: "512m",
+                        NEO4J_server_memory_heap_max__size: "512m",
         //		NEO4J_server_bolt_listen__address: "0.0.0.0:7687",
         //		NEO4J_server_default__listen__address: "0.0.0.0",
         //		NEO4J_server_http_listen__address: "0.0.0.0:7474",
                     })
-                    .with_limits("1.0", "768M")
+                    .with_limits("1.0", "1536M")
-                    .with_reservations("0.5", "768M")
+                    .with_reservations("0.5", "1536M")
                     .with_port(7474, 7474, "api")
                     .with_port(7687, 7687, "api2")
                     .with_volume_mount(vol, "/data");

templates/values/images.jsonnet

@@ -1,7 +1,7 @@
 local version = import "version.jsonnet";
 {
     cassandra: "docker.io/cassandra:4.1.6",
-    neo4j: "docker.io/neo4j:5.22.0-community-bullseye",
+    neo4j: "docker.io/neo4j:5.26.0-community-bullseye",
     pulsar: "docker.io/apachepulsar/pulsar:3.3.1",
     pulsar_manager: "docker.io/apachepulsar/pulsar-manager:v0.4.0",
     etcd: "quay.io/coreos/etcd:v3.5.15",
@@ -9,10 +9,14 @@ local version = import "version.jsonnet";
     milvus: "docker.io/milvusdb/milvus:v2.4.9",
     prometheus: "docker.io/prom/prometheus:v2.53.2",
     grafana: "docker.io/grafana/grafana:11.1.4",
-    trustgraph: "docker.io/trustgraph/trustgraph-flow:" + version,
+    trustgraph_base: "docker.io/trustgraph/trustgraph-base:" + version,
+    trustgraph_flow: "docker.io/trustgraph/trustgraph-flow:" + version,
+    trustgraph_bedrock: "docker.io/trustgraph/trustgraph-bedrock:" + version,
+    trustgraph_vertexai: "docker.io/trustgraph/trustgraph-vertexai:" + version,
+    trustgraph_hf: "docker.io/trustgraph/trustgraph-hf:" + version,
     qdrant: "docker.io/qdrant/qdrant:v1.11.1",
     memgraph_mage: "docker.io/memgraph/memgraph-mage:1.22-memgraph-2.22",
     memgraph_lab: "docker.io/memgraph/lab:2.19.1",
     falkordb: "docker.io/falkordb/falkordb:latest",
-    "workbench-ui": "docker.io/trustgraph/workbench-ui:0.1.6",
+    "workbench-ui": "docker.io/trustgraph/workbench-ui:0.2.4",
 }

trustgraph-base/trustgraph/api/api.py

@@ -131,6 +131,35 @@ class Api:
         except:
             raise ProtocolException(f"Response not formatted correctly")
 
+    def document_rag(self, question):
+
+        # The input consists of a question
+        input = {
+            "query": question
+        }
+
+        url = f"{self.url}document-rag"
+
+        # Invoke the API, input is passed as JSON
+        resp = requests.post(url, json=input)
+
+        # Should be a 200 status code
+        if resp.status_code != 200:
+            raise ProtocolException(f"Status code {resp.status_code}")
+
+        try:
+            # Parse the response as JSON
+            object = resp.json()
+        except:
+            raise ProtocolException(f"Expected JSON response")
+
+        self.check_error(resp)
+
+        try:
+            return object["response"]
+        except:
+            raise ProtocolException(f"Response not formatted correctly")
+
     def embeddings(self, text):
 
         # The input consists of a text block

trustgraph-base/trustgraph/base/consumer.py

@@ -1,11 +1,15 @@
 
 from pulsar.schema import JsonSchema
+import pulsar
 from prometheus_client import Histogram, Info, Counter, Enum
 import time
 
 from . base_processor import BaseProcessor
 from .. exceptions import TooManyRequests
 
+default_rate_limit_retry = 10
+default_rate_limit_timeout = 7200
+
 class Consumer(BaseProcessor):
 
     def __init__(self, **params):
@@ -21,11 +25,18 @@ class Consumer(BaseProcessor):
 
         super(Consumer, self).__init__(**params)
 
-        input_queue = params.get("input_queue")
+        self.input_queue = params.get("input_queue")
-        subscriber = params.get("subscriber")
+        self.subscriber = params.get("subscriber")
-        input_schema = params.get("input_schema")
+        self.input_schema = params.get("input_schema")
 
-        if input_schema == None:
+        self.rate_limit_retry = params.get(
+            "rate_limit_retry", default_rate_limit_retry
+        )
+        self.rate_limit_timeout = params.get(
+            "rate_limit_timeout", default_rate_limit_timeout
+        )
+
+        if self.input_schema == None:
             raise RuntimeError("input_schema must be specified")
 
         if not hasattr(__class__, "request_metric"):
@@ -43,17 +54,27 @@ class Consumer(BaseProcessor):
                 'processing_count', 'Processing count', ["status"]
             )
 
+        if not hasattr(__class__, "rate_limit_metric"):
+            __class__.rate_limit_metric = Counter(
+                'rate_limit_count', 'Rate limit event count',
+            )
+
         __class__.pubsub_metric.info({
-            "input_queue": input_queue,
+            "input_queue": self.input_queue,
-            "subscriber": subscriber,
+            "subscriber": self.subscriber,
-            "input_schema": input_schema.__name__,
+            "input_schema": self.input_schema.__name__,
+            "rate_limit_retry": str(self.rate_limit_retry),
+            "rate_limit_timeout": str(self.rate_limit_timeout),
         })
 
         self.consumer = self.client.subscribe(
-            input_queue, subscriber,
+            self.input_queue, self.subscriber,
-            schema=JsonSchema(input_schema),
+            consumer_type=pulsar.ConsumerType.Shared,
+            schema=JsonSchema(self.input_schema),
         )
 
+        print("Initialised consumer.", flush=True)
+
     def run(self):
 
         __class__.state_metric.state('running')
@@ -62,31 +83,61 @@ class Consumer(BaseProcessor):
 
             msg = self.consumer.receive()
 
-            try:
+            expiry = time.time() + self.rate_limit_timeout
 
-                with __class__.request_metric.time():
+            # This loop is for retry on rate-limit / resource limits
-                    self.handle(msg)
+            while True:
 
-                # Acknowledge successful processing of the message
+                if time.time() > expiry:
-                self.consumer.acknowledge(msg)
 
-                __class__.processing_metric.labels(status="success").inc()
+                    print("Gave up waiting for rate-limit retry", flush=True)
 
-            except TooManyRequests:
+                    # Message failed to be processed, this causes it to
-                self.consumer.negative_acknowledge(msg)
+                    # be retried
-                print("TooManyRequests: will retry")
+                    self.consumer.negative_acknowledge(msg)
-                __class__.processing_metric.labels(status="rate-limit").inc()
+
-                time.sleep(5)
+                    __class__.processing_metric.labels(status="error").inc()
-                continue
+
+                    # Break out of retry loop, processes next message
+                    break
+
+                try:
+
+                    with __class__.request_metric.time():
+                        self.handle(msg)
+
+                    # Acknowledge successful processing of the message
+                    self.consumer.acknowledge(msg)
+
+                    __class__.processing_metric.labels(status="success").inc()
+
+                    # Break out of retry loop
+                    break
+
+                except TooManyRequests:
+
+                    print("TooManyRequests: will retry...", flush=True)
+
+                    __class__.rate_limit_metric.inc()
+
+                    # Sleep
+                    time.sleep(self.rate_limit_retry)
+
+                    # Contine from retry loop, just causes a reprocessing
+                    continue
                 
-            except Exception as e:
+                except Exception as e:
 
-                print("Exception:", e, flush=True)
+                    print("Exception:", e, flush=True)
 
-                # Message failed to be processed
+                    # Message failed to be processed, this causes it to
-                self.consumer.negative_acknowledge(msg)
+                    # be retried
+                    self.consumer.negative_acknowledge(msg)
 
-                __class__.processing_metric.labels(status="error").inc()
+                    __class__.processing_metric.labels(status="error").inc()
+
+                    # Break out of retry loop, processes next message
+                    break
 
     @staticmethod
     def add_args(parser, default_input_queue, default_subscriber):
@@ -105,3 +156,17 @@ class Consumer(BaseProcessor):
             help=f'Queue subscriber name (default: {default_subscriber})'
         )
 
+        parser.add_argument(
+            '--rate-limit-retry',
+            type=int,
+            default=default_rate_limit_retry,
+            help=f'Rate limit retry (default: {default_rate_limit_retry})'
+        )
+
+        parser.add_argument(
+            '--rate-limit-timeout',
+            type=int,
+            default=default_rate_limit_timeout,
+            help=f'Rate limit timeout (default: {default_rate_limit_timeout})'
+        )
+

trustgraph-base/trustgraph/base/consumer_producer.py

@@ -1,111 +1,46 @@
 
 from pulsar.schema import JsonSchema
+import pulsar
 from prometheus_client import Histogram, Info, Counter, Enum
 import time
 
-from . base_processor import BaseProcessor
+from . consumer import Consumer
 from .. exceptions import TooManyRequests
 
-# FIXME: Derive from consumer?  And producer?
+class ConsumerProducer(Consumer):
-
-class ConsumerProducer(BaseProcessor):
 
     def __init__(self, **params):
 
-        if not hasattr(__class__, "state_metric"):
+        super(ConsumerProducer, self).__init__(**params)
-            __class__.state_metric = Enum(
-                'processor_state', 'Processor state',
-                states=['starting', 'running', 'stopped']
-            )
-            __class__.state_metric.state('starting')
 
-        __class__.state_metric.state('starting')
+        self.output_queue = params.get("output_queue")
-
+        self.output_schema = params.get("output_schema")
-        input_queue = params.get("input_queue")
-        output_queue = params.get("output_queue")
-        subscriber = params.get("subscriber")
-        input_schema = params.get("input_schema")
-        output_schema = params.get("output_schema")
-
-        if not hasattr(__class__, "request_metric"):
-            __class__.request_metric = Histogram(
-                'request_latency', 'Request latency (seconds)'
-            )
 
         if not hasattr(__class__, "output_metric"):
             __class__.output_metric = Counter(
                 'output_count', 'Output items created'
             )
 
-        if not hasattr(__class__, "pubsub_metric"):
-            __class__.pubsub_metric = Info(
-                'pubsub', 'Pub/sub configuration'
-            )
-
-        if not hasattr(__class__, "processing_metric"):
-            __class__.processing_metric = Counter(
-                'processing_count', 'Processing count', ["status"]
-            )
-
         __class__.pubsub_metric.info({
-            "input_queue": input_queue,
+            "input_queue": self.input_queue,
-            "output_queue": output_queue,
+            "output_queue": self.output_queue,
-            "subscriber": subscriber,
+            "subscriber": self.subscriber,
-            "input_schema": input_schema.__name__,
+            "input_schema": self.input_schema.__name__,
-            "output_schema": output_schema.__name__,
+            "output_schema": self.output_schema.__name__,
+            "rate_limit_retry": str(self.rate_limit_retry),
+            "rate_limit_timeout": str(self.rate_limit_timeout),
         })
 
-        super(ConsumerProducer, self).__init__(**params)
+        if self.output_schema == None:
-
-        if input_schema == None:
-            raise RuntimeError("input_schema must be specified")
-
-        if output_schema == None:
             raise RuntimeError("output_schema must be specified")
 
         self.producer = self.client.create_producer(
-            topic=output_queue,
+            topic=self.output_queue,
-            schema=JsonSchema(output_schema),
+            schema=JsonSchema(self.output_schema),
+            chunking_enabled=True,
         )
 
-        self.consumer = self.client.subscribe(
+        print("Initialised consumer/producer.")
-            input_queue, subscriber,
-            schema=JsonSchema(input_schema),
-        )
-
-    def run(self):
-
-        __class__.state_metric.state('running')
-
-        while True:
-
-            msg = self.consumer.receive()
-
-            try:
-
-                with __class__.request_metric.time():
-                    resp = self.handle(msg)
-
-                # Acknowledge successful processing of the message
-                self.consumer.acknowledge(msg)
-
-                __class__.processing_metric.labels(status="success").inc()
-
-            except TooManyRequests:
-                self.consumer.negative_acknowledge(msg)
-                print("TooManyRequests: will retry")
-                __class__.processing_metric.labels(status="rate-limit").inc()
-                time.sleep(5)
-                continue
-
-            except Exception as e:
-
-                print("Exception:", e, flush=True)
-
-                # Message failed to be processed
-                self.consumer.negative_acknowledge(msg)
-
-                __class__.processing_metric.labels(status="error").inc()
 
     def send(self, msg, properties={}):
         self.producer.send(msg, properties)
@@ -117,19 +52,7 @@ class ConsumerProducer(BaseProcessor):
             default_output_queue,
     ):
 
-        BaseProcessor.add_args(parser)
+        Consumer.add_args(parser, default_input_queue, default_subscriber)
-
-        parser.add_argument(
-            '-i', '--input-queue',
-            default=default_input_queue,
-            help=f'Input queue (default: {default_input_queue})'
-        )
-
-        parser.add_argument(
-            '-s', '--subscriber',
-            default=default_subscriber,
-            help=f'Queue subscriber name (default: {default_subscriber})'
-        )
 
         parser.add_argument(
             '-o', '--output-queue',

trustgraph-base/trustgraph/base/producer.py

@@ -34,6 +34,7 @@ class Producer(BaseProcessor):
         self.producer = self.client.create_producer(
             topic=output_queue,
             schema=JsonSchema(output_schema),
+            chunking_enabled=True,
         )
 
     def send(self, msg, properties={}):

trustgraph-base/trustgraph/clients/document_embeddings_client.py

@@ -40,8 +40,12 @@ class DocumentEmbeddingsClient(BaseClient):
             output_schema=DocumentEmbeddingsResponse,
         )
 
-    def request(self, vectors, limit=10, timeout=300):
+    def request(
+            self, vectors, user="trustgraph", collection="default",
+            limit=10, timeout=300
+    ):
         return self.call(
+            user=user, collection=collection,
             vectors=vectors, limit=limit, timeout=timeout
         ).documents
 

trustgraph-base/trustgraph/exceptions.py

@@ -8,7 +8,3 @@ class LlmError(Exception):
 class ParseError(Exception):
     pass
 
-
-
-
-

trustgraph-base/trustgraph/schema/documents.py

@@ -35,14 +35,18 @@ chunk_ingest_queue = topic('chunk-load')
 
 ############################################################################
 
-# Chunk embeddings are an embeddings associated with a text chunk
+# Document embeddings are embeddings associated with a chunk
 
 class ChunkEmbeddings(Record):
-    metadata = Metadata()
-    vectors = Array(Array(Double()))
     chunk = Bytes()
+    vectors = Array(Array(Double()))
 
-chunk_embeddings_ingest_queue = topic('chunk-embeddings-load')
+# This is a 'batching' mechanism for the above data
+class DocumentEmbeddings(Record):
+    metadata = Metadata()
+    chunks = Array(ChunkEmbeddings())
+
+document_embeddings_store_queue = topic('document-embeddings-store')
 
 ############################################################################
 
@@ -51,6 +55,8 @@ chunk_embeddings_ingest_queue = topic('chunk-embeddings-load')
 class DocumentEmbeddingsRequest(Record):
     vectors = Array(Array(Double()))
     limit = Integer()
+    user = String()
+    collection = String()
 
 class DocumentEmbeddingsResponse(Record):
     error = Error()
@@ -62,3 +68,4 @@ document_embeddings_request_queue = topic(
 document_embeddings_response_queue = topic(
     'doc-embeddings', kind='non-persistent', namespace='response',
 )
+

trustgraph-base/trustgraph/schema/graph.py

@@ -7,12 +7,31 @@ from . metadata import Metadata
 
 ############################################################################
 
+# Entity context are an entity associated with textual context
+
+class EntityContext(Record):
+    entity = Value()
+    context = String()
+
+# This is a 'batching' mechanism for the above data
+class EntityContexts(Record):
+    metadata = Metadata()
+    entities = Array(EntityContext())
+
+entity_contexts_ingest_queue = topic('entity-contexts-load')
+
+############################################################################
+
 # Graph embeddings are embeddings associated with a graph entity
 
+class EntityEmbeddings(Record):
+    entity = Value()
+    vectors = Array(Array(Double()))
+
+# This is a 'batching' mechanism for the above data
 class GraphEmbeddings(Record):
     metadata = Metadata()
-    vectors = Array(Array(Double()))
+    entities = Array(EntityEmbeddings())
-    entity = Value()
 
 graph_embeddings_store_queue = topic('graph-embeddings-store')
 

trustgraph-bedrock/setup.py

@@ -34,7 +34,7 @@ setuptools.setup(
     python_requires='>=3.8',
     download_url = "https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v" + version + ".tar.gz",
     install_requires=[
-        "trustgraph-base>=0.18,<0.19",
+        "trustgraph-base>=0.21,<0.22",
         "pulsar-client",
         "prometheus-client",
         "boto3",

trustgraph-bedrock/trustgraph/model/text_completion/bedrock/llm.py

@@ -24,32 +24,48 @@ default_subscriber = module
 default_model = 'mistral.mistral-large-2407-v1:0'
 default_temperature = 0.0
 default_max_output = 2048
-default_aws_id_key = os.getenv("AWS_ID_KEY", None)
+
-default_aws_secret = os.getenv("AWS_SECRET", None)
+# Actually, these could all just be None, no need to get environment
-default_aws_region = os.getenv("AWS_REGION", 'us-west-2')
+# variables, as Boto3 would pick all these up if not passed in as args
+default_access_key_id = os.getenv("AWS_ACCESS_KEY_ID", None)
+default_secret_access_key = os.getenv("AWS_SECRET_ACCESS_KEY", None)
+default_session_token = os.getenv("AWS_SESSION_TOKEN", None)
+default_profile = os.getenv("AWS_PROFILE", None)
+default_region = os.getenv("AWS_DEFAULT_REGION", None)
 
 class Processor(ConsumerProducer):
 
     def __init__(self, **params):
+
+        print(params)
     
         input_queue = params.get("input_queue", default_input_queue)
         output_queue = params.get("output_queue", default_output_queue)
         subscriber = params.get("subscriber", default_subscriber)
+
         model = params.get("model", default_model)
-        aws_id_key = params.get("aws_id_key", default_aws_id_key)
-        aws_secret = params.get("aws_secret", default_aws_secret)
-        aws_region = params.get("aws_region", default_aws_region)
         temperature = params.get("temperature", default_temperature)
         max_output = params.get("max_output", default_max_output)
 
-        if aws_id_key is None:
+        aws_access_key_id = params.get(
-            raise RuntimeError("AWS ID not specified")
+            "aws_access_key_id", default_access_key_id
+        )
 
-        if aws_secret is None:
+        aws_secret_access_key = params.get(
-            raise RuntimeError("AWS secret not specified")
+            "aws_secret_access_key", default_secret_access_key
+        )
 
-        if aws_region is None:
+        aws_session_token = params.get(
-            raise RuntimeError("AWS region not specified")
+            "aws_session_token", default_session_token
+        )
+
+        aws_region = params.get(
+            "aws_region", default_region
+        )
+
+        aws_profile = params.get(
+            "aws_profile", default_profile
+        )
 
         super(Processor, self).__init__(
             **params | {
@@ -82,9 +98,11 @@ class Processor(ConsumerProducer):
         self.max_output = max_output
 
         self.session = boto3.Session(
-            aws_access_key_id=aws_id_key,
+            aws_access_key_id=aws_access_key_id,
-            aws_secret_access_key=aws_secret,
+            aws_secret_access_key=aws_secret_access_key,
-            region_name=aws_region
+            aws_session_token=aws_session_token,
+            profile_name=aws_profile,
+            region_name=aws_region,
         )
 
         self.bedrock = self.session.client(service_name='bedrock-runtime')
@@ -179,9 +197,6 @@ class Processor(ConsumerProducer):
             accept = 'application/json'
             contentType = 'application/json'
 
-            # FIXME: Consider catching request limits and raise TooManyRequests
-            # See https://boto3.amazonaws.com/v1/documentation/api/latest/guide/retries.html
-
             with __class__.text_completion_metric.time():
                 response = self.bedrock.invoke_model(
                     body=promptbody, modelId=self.model, accept=accept,
@@ -243,30 +258,18 @@ class Processor(ConsumerProducer):
 
             print("Done.", flush=True)
 
+        except self.bedrock.exceptions.ThrottlingException as e:
 
-        # FIXME: Wrong exception, don't know what Bedrock throws
+            print("Hit rate limit:", e, flush=True)
-        # for a rate limit
-        except TooManyRequests:
 
-            print("Send rate limit response...", flush=True)
+            # Leave rate limit retries to the base handler
-
+            raise TooManyRequests()
-            r = TextCompletionResponse(
-                error=Error(
-                    type = "rate-limit",
-                    message = str(e),
-                ),
-                response=None,
-                in_token=None,
-                out_token=None,
-                model=None,
-            )
-
-            self.producer.send(r, properties={"id": id})
-
-            self.consumer.acknowledge(msg)
 
         except Exception as e:
 
+            # Apart from rate limits, treat all exceptions as unrecoverable
+
+            print(type(e))
             print(f"Exception: {e}")
 
             print("Send error response...", flush=True)
@@ -299,21 +302,27 @@ class Processor(ConsumerProducer):
         )
 
         parser.add_argument(
-            '-z', '--aws-id-key',
+            '-z', '--aws-access-key-id',
-            default=default_aws_id_key,
+            default=default_access_key_id,
-            help=f'AWS ID Key'
+            help=f'AWS access key ID'
         )
 
         parser.add_argument(
-            '-k', '--aws-secret',
+            '-k', '--aws-secret-access-key',
-            default=default_aws_secret,
+            default=default_secret_access_key,
-            help=f'AWS Secret Key'
+            help=f'AWS secret access key'
         )
 
         parser.add_argument(
             '-r', '--aws-region',
-            default=default_aws_region,
+            default=default_region,
-            help=f'AWS Region'
+            help=f'AWS region'
+        )
+
+        parser.add_argument(
+            '--aws-profile', '--profile', 
+            default=default_profile,
+            help=f'AWS profile name'
         )
 
         parser.add_argument(

trustgraph-cli/scripts/tg-dump-msgpack

@@ -9,6 +9,7 @@ diagnostic utility.
 import msgpack
 import sys
 import argparse
+import json
 
 def dump(input_file, action):
 
@@ -17,7 +18,7 @@ def dump(input_file, action):
         unpacker = msgpack.Unpacker(f, raw=False)
 
         for unpacked in unpacker:
-            print(unpacked)
+            print(json.dumps(unpacked))
 
 def summary(input_file, action):
 

trustgraph-cli/scripts/tg-graph-show

@@ -6,24 +6,23 @@ Connects to the graph query service and dumps all graph edges.
 
 import argparse
 import os
-from trustgraph.clients.triples_query_client import TriplesQueryClient
+from trustgraph.api import Api
 
-default_pulsar_host = os.getenv("PULSAR_HOST", 'pulsar://localhost:6650')
+default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
-default_pulsar_api_key = os.getenv("PULSAR_API_KEY", None)
 default_user = 'trustgraph'
 default_collection = 'default'
 
-def show_graph(pulsar, user, collection, pulsar_api_key=None):
+def show_graph(url, user, collection):
 
-    tq = TriplesQueryClient(pulsar_host=pulsar, pulsar_api_key=pulsar_api_key)
+    api = Api(url)
 
-    rows = tq.request(
+    rows = api.triples_query(
-        user=user, collection=collection,
+#         user=user, collection=collection,
-        s=None, p=None, o=None, limit=10_000_000
+        s=None, p=None, o=None, limit=10_000,
     )
 
     for row in rows:
-        print(row.s.value, row.p.value, row.o.value)
+        print(row.s, row.p, row.o)
 
 def main():
 
@@ -33,35 +32,30 @@ def main():
     )
 
     parser.add_argument(
-        '-p', '--pulsar-host',
+        '-u', '--api-url',
-        default=default_pulsar_host,
+        default=default_url,
-        help=f'Pulsar host (default: {default_pulsar_host})',
+        help=f'API URL (default: {default_url})',
     )
 
     parser.add_argument(
-        '-u', '--user',
+        '-U', '--user',
         default=default_user,
         help=f'User ID (default: {default_user})'
     )
 
     parser.add_argument(
-        '-c', '--collection',
+        '-C', '--collection',
         default=default_collection,
         help=f'Collection ID (default: {default_collection})'
     )
-        
-    parser.add_argument(
-        '--pulsar-api-key',
-        default=default_pulsar_api_key,
-        help=f'Pulsar API key',
-    )
     
     args = parser.parse_args()
 
     try:
 
         show_graph(
-            pulsar=args.pulsar_host, user=args.user,
+            url=args.api_url,
+            user=args.user,
             collection=args.collection,
             pulsar_api_key=args.pulsar_api_key,
         )

trustgraph-cli/scripts/tg-graph-to-turtle

@@ -5,38 +5,45 @@ Connects to the graph query service and dumps all graph edges in Turtle
 format.
 """
 
-import argparse
-import os
-from trustgraph.clients.triples_query_client import TriplesQueryClient
 import rdflib
 import io
 import sys
+import argparse
+import os
 
-default_pulsar_host = os.getenv("PULSAR_HOST", 'pulsar://localhost:6650')
+from trustgraph.api import Api, Uri
-default_pulsar_api_key = os.getenv("PULSAR_API_KEY", None)
 
-def show_graph(pulsar, pulsar_api_key=None):
+default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
+default_user = 'trustgraph'
+default_collection = 'default'
 
-    tq = TriplesQueryClient(pulsar_host=pulsar, pulsar_api_key=pulsar_api_key)
+def show_graph(url, user, collection):
 
-    rows = tq.request(None, None, None, limit=10_000_000)
+    api = Api(url)
+
+    rows = api.triples_query(
+        s=None, p=None, o=None,
+        limit=10_000)
+#         user=user, collection=collection,
 
     g = rdflib.Graph()
 
     for row in rows:
 
-        sv = rdflib.term.URIRef(row.s.value)
+        sv = rdflib.term.URIRef(row.s)
-        pv = rdflib.term.URIRef(row.p.value)
+        pv = rdflib.term.URIRef(row.p)
 
-        if row.o.is_uri:
+        if isinstance(row.o, Uri):
 
             # Skip malformed URLs with spaces in
-            if " " in row.o.value:
+            if " " in row.o:
                 continue
 
-            ov = rdflib.term.URIRef(row.o.value)
+            ov = rdflib.term.URIRef(row.o)
+
         else:
-            ov = rdflib.term.Literal(row.o.value)
+
+            ov = rdflib.term.Literal(row.o)
 
         g.add((sv, pv, ov))
 
@@ -57,22 +64,32 @@ def main():
     )
 
     parser.add_argument(
-        '-p', '--pulsar-host',
+        '-u', '--api-url',
-        default=default_pulsar_host,
+        default=default_url,
-        help=f'Pulsar host (default: {default_pulsar_host})',
+        help=f'API URL (default: {default_url})',
     )
-    
+
     parser.add_argument(
-        '--pulsar-api-key',
+        '-U', '--user',
-        default=default_pulsar_api_key,
+        default=default_user,
-        help=f'Pulsar API key',
+        help=f'User ID (default: {default_user})'
+    )
+
+    parser.add_argument(
+        '-C', '--collection',
+        default=default_collection,
+        help=f'Collection ID (default: {default_collection})'
     )
     
     args = parser.parse_args()
 
     try:
 
-        show_graph(args.pulsar_host, pulsar_api_key=args.pulsar_api_key)
+        show_graph(
+            url=args.api_url,
+            user=args.user,
+            collection=args.collection
+        )
 
     except Exception as e:
 

trustgraph-cli/scripts/tg-invoke-agent

@@ -1,17 +1,18 @@
 #!/usr/bin/env python3
 
 """
-Uses the GraphRAG service to answer a query
+Uses the GraphRAG service to answer a question
 """
 
 import argparse
 import os
 import textwrap
+import uuid
+import asyncio
+import json
+from websockets.asyncio.client import connect
 
-from trustgraph.clients.agent_client import AgentClient
+default_url = os.getenv("TRUSTGRAPH_URL", 'ws://localhost:8088/')
-
-default_pulsar_host = os.getenv("PULSAR_HOST", 'pulsar://localhost:6650')
-default_pulsar_api_key = os.getenv("PULSAR_API_KEY", None)
 default_user = 'trustgraph'
 default_collection = 'default'
 
@@ -28,15 +29,18 @@ def output(text, prefix="> ", width=78):
     )
     print(out)
 
-def query(
+async def question(
-        pulsar_host, query, user, collection,
+        url, question, user, collection,
-        plan=None, state=None, verbose=False, pulsar_api_key=None
+        plan=None, state=None, verbose=False
 ):
 
-    am = AgentClient(pulsar_host=pulsar_host, pulsar_api_key=pulsar_api_key)
+    if not url.endswith("/"):
+        url += "/"
+
+    url = url + "api/v1/socket"
 
     if verbose:
-        output(wrap(query), "\U00002753 ")
+        output(wrap(question), "\U00002753 ")
         print()
 
     def think(x):
@@ -49,11 +53,43 @@ def query(
             output(wrap(x), "\U0001f4a1 ")
             print()
 
-    resp = am.request(
+    mid = str(uuid.uuid4())
-        question=query, think=think, observe=observe,
-    )
 
-    print(resp)
+    async with connect(url) as ws:
+
+        req = json.dumps({
+            "id": mid,
+            "service": "agent",
+            "request": {
+                "question": question,
+            }
+            
+        })
+
+        await ws.send(req)
+
+        while True:
+
+            msg = await ws.recv()
+
+            obj = json.loads(msg)
+
+            if obj["id"] != mid:
+                print("Ignore message")
+                continue
+
+            if "thought" in obj["response"]:
+                think(obj["response"]["thought"])
+
+            if "observation" in obj["response"]:
+                observe(obj["response"]["observation"])
+
+            if "answer" in obj["response"]:
+                print(obj["response"]["answer"])
+
+            if obj["complete"]: break
+
+        await ws.close()
 
 def main():
 
@@ -63,25 +99,25 @@ def main():
     )
 
     parser.add_argument(
-        '-p', '--pulsar-host',
+        '-u', '--url',
-        default=default_pulsar_host,
+        default=default_url,
-        help=f'Pulsar host (default: {default_pulsar_host})',
+        help=f'API URL (default: {default_url})',
     )
 
     parser.add_argument(
-        '-q', '--query',
+        '-q', '--question',
         required=True,
-        help=f'Query to execute',
+        help=f'Question to answer',
     )
 
     parser.add_argument(
-        '-u', '--user',
+        '-U', '--user',
         default=default_user,
         help=f'User ID (default: {default_user})'
     )
 
     parser.add_argument(
-        '-c', '--collection',
+        '-C', '--collection',
         default=default_collection,
         help=f'Collection ID (default: {default_collection})'
     )
@@ -101,26 +137,21 @@ def main():
         action="store_true",
         help=f'Output thinking/observations'
     )
-    
-    parser.add_argument(
-        '--pulsar-api-key',
-        default=default_pulsar_api_key,
-        help=f'Pulsar API key',
-    )
 
     args = parser.parse_args()
 
     try:
 
-        query(
+        asyncio.run(
-            pulsar_host=args.pulsar_host,
+            question(
-            query=args.query,
+                url=args.url,
-            user=args.user,
+                question=args.question,
-            collection=args.collection,
+                user=args.user,
-            plan=args.plan,
+                collection=args.collection,
-            state=args.state,
+                plan=args.plan,
-            verbose=args.verbose,
+                state=args.state,
-            pulsar_api_key=args.pulsar_api_key,
+                verbose=args.verbose,
+            )
         )
 
     except Exception as e:

trustgraph-cli/scripts/tg-invoke-document-rag

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+
+"""
+Uses the GraphRAG service to answer a question
+"""
+
+import argparse
+import os
+from trustgraph.api import Api
+
+default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
+default_user = 'trustgraph'
+default_collection = 'default'
+
+def question(url, question, user, collection):
+
+    rag = Api(url)
+
+# user=user, collection=collection, 
+    resp = rag.document_rag(question=question)
+
+    print(resp)
+
+def main():
+
+    parser = argparse.ArgumentParser(
+        prog='tg-invoke-document-rag',
+        description=__doc__,
+    )
+
+    parser.add_argument(
+        '-u', '--url',
+        default=default_url,
+        help=f'API URL (default: {default_url})',
+    )
+
+    parser.add_argument(
+        '-q', '--question',
+        required=True,
+        help=f'Question to answer',
+    )
+
+    parser.add_argument(
+        '-U', '--user',
+        default=default_user,
+        help=f'User ID (default: {default_user})'
+    )
+
+    parser.add_argument(
+        '-C', '--collection',
+        default=default_collection,
+        help=f'Collection ID (default: {default_collection})'
+    )
+
+    args = parser.parse_args()
+
+    try:
+
+        question(
+            url=args.url,
+            question=args.question,
+            user=args.user,
+            collection=args.collection,
+        )
+
+    except Exception as e:
+
+        print("Exception:", e, flush=True)
+
+main()
+

trustgraph-cli/scripts/tg-invoke-graph-rag

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+
+"""
+Uses the GraphRAG service to answer a question
+"""
+
+import argparse
+import os
+from trustgraph.api import Api
+
+default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
+default_user = 'trustgraph'
+default_collection = 'default'
+
+def question(url, question, user, collection):
+
+    rag = Api(url)
+
+# user=user, collection=collection, 
+    resp = rag.graph_rag(question=question)
+
+    print(resp)
+
+def main():
+
+    parser = argparse.ArgumentParser(
+        prog='tg-invoke-graph-rag',
+        description=__doc__,
+    )
+
+    parser.add_argument(
+        '-u', '--url',
+        default=default_url,
+        help=f'API URL (default: {default_url})',
+    )
+    
+    parser.add_argument(
+        '-q', '--question',
+        required=True,
+        help=f'Question to answer',
+    )
+
+    parser.add_argument(
+        '-U', '--user',
+        default=default_user,
+        help=f'User ID (default: {default_user})'
+    )
+
+    parser.add_argument(
+        '-C', '--collection',
+        default=default_collection,
+        help=f'Collection ID (default: {default_collection})'
+    )
+
+    args = parser.parse_args()
+
+    try:
+
+        question(
+            url=args.url,
+            question=args.question,
+            user=args.user,
+            collection=args.collection,
+        )
+
+    except Exception as e:
+
+        print("Exception:", e, flush=True)
+
+main()
+

trustgraph-cli/scripts/tg-invoke-llm

@@ -8,17 +8,15 @@ and user prompt.  Both arguments are required.
 import argparse
 import os
 import json
-from trustgraph.clients.llm_client import LlmClient
+from trustgraph.api import Api
 
-default_pulsar_host = os.getenv("PULSAR_HOST", 'pulsar://localhost:6650')
+default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
-default_pulsar_api_key = os.getenv("PULSAR_API_KEY", None)
 
+def query(url, system, prompt):
 
-def query(pulsar_host, system, prompt, pulsar_api_key=None):
+    api = Api(url)
 
-    cli = LlmClient(pulsar_host=pulsar_host, pulsar_api_key=pulsar_api_key)
+    resp = api.text_completion(system=system, prompt=prompt)
-
-    resp = cli.request(system=system, prompt=prompt)
 
     print(resp)
 
@@ -30,9 +28,9 @@ def main():
     )
 
     parser.add_argument(
-        '-p', '--pulsar-host',
+        '-u', '--url',
-        default=default_pulsar_host,
+        default=default_url,
-        help=f'Pulsar host (default: {default_pulsar_host})',
+        help=f'API URL (default: {default_url})',
     )
     
     parser.add_argument(
@@ -46,23 +44,15 @@ def main():
         nargs=1,
         help='LLM prompt e.g. What is 2 + 2?',
     )
-    
-    parser.add_argument(
-        '--pulsar-api-key',
-        default=default_pulsar_api_key,
-        help=f'Pulsar API key',
-    )
-
 
     args = parser.parse_args()
 
     try:
 
         query(
-            pulsar_host=args.pulsar_host,
+            url=args.url,
             system=args.system[0],
             prompt=args.prompt[0],
-            pulsar_api_key=args.pulsar_api_key,
         )
 
     except Exception as e:

trustgraph-cli/scripts/tg-invoke-prompt

@@ -12,17 +12,15 @@ using key=value arguments on the command line, and these replace
 import argparse
 import os
 import json
-from trustgraph.clients.prompt_client import PromptClient
+from trustgraph.api import Api
 
-default_pulsar_host = os.getenv("PULSAR_HOST", 'pulsar://localhost:6650')
+default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
-default_pulsar_api_key = os.getenv("PULSAR_API_KEY", None)
 
+def query(url, template_id, variables):
 
-def query(pulsar_host, template_id, variables, pulsar_api_key=None):
+    api = Api(url)
 
-    cli = PromptClient(pulsar_host=pulsar_host, pulsar_api_key=pulsar_api_key)
+    resp = api.prompt(id=template_id, variables=variables)
-
-    resp = cli.request(id=template_id, variables=variables)
 
     if isinstance(resp, str):
         print(resp)
@@ -37,9 +35,9 @@ def main():
     )
 
     parser.add_argument(
-        '-p', '--pulsar-host',
+        '-u', '--url',
-        default=default_pulsar_host,
+        default=default_url,
-        help=f'Pulsar host (default: {default_pulsar_host})',
+        help=f'API URL (default: {default_url})',
     )
 
     parser.add_argument(
@@ -57,13 +55,6 @@ def main():
 specified multiple times''',
     )
 
-    
-    parser.add_argument(
-        '--pulsar-api-key',
-        default=default_pulsar_api_key,
-        help=f'Pulsar API key',
-    )
-
     args = parser.parse_args()
 
     variables = {}
@@ -79,10 +70,9 @@ specified multiple times''',
     try:
 
         query(
-            pulsar_host=args.pulsar_host,
+            url=args.url,
             template_id=args.id[0],
             variables=variables,
-            pulsar_api_key=args.pulsar_api_key,
         )
 
     except Exception as e:

trustgraph-cli/scripts/tg-load-doc-embeds

@@ -0,0 +1,224 @@
+#!/usr/bin/env python3
+
+"""This utility takes a knowledge core and loads it into a running TrustGraph
+through the API.  The knowledge core should be in msgpack format, which is the
+default format produce by tg-save-kg-core.
+"""
+
+import aiohttp
+import asyncio
+import msgpack
+import json
+import sys
+import argparse
+import os
+import signal
+
+class Running:
+    def __init__(self): self.running = True
+    def get(self): return self.running
+    def stop(self): self.running = False
+
+de_counts = 0
+
+async def load_de(running, queue, url):
+
+    global de_counts 
+
+    async with aiohttp.ClientSession() as session:
+
+        async with session.ws_connect(f"{url}load/document-embeddings") as ws:
+
+            while running.get():
+
+                try:
+                    msg = await asyncio.wait_for(queue.get(), 1)
+
+                    # End of load
+                    if msg is None:
+                        break
+
+                except:
+                    # Hopefully it's TimeoutError.  Annoying to match since
+                    # it changed in 3.11.
+                    continue
+
+                msg = {
+                    "metadata": {
+                        "id": msg["m"]["i"], 
+                        "metadata": msg["m"]["m"],
+                        "user": msg["m"]["u"],
+                        "collection": msg["m"]["c"],
+                    },
+                    "chunks": [
+                        {
+                            "chunk": chunk["c"],
+                            "vectors": chunk["v"],
+                        }
+                        for chunk in msg["c"]
+                    ],
+                }
+
+                try:
+                    await ws.send_json(msg)
+                except Exception as e:
+                    print(e)
+
+                de_counts += 1
+
+async def stats(running):
+
+    global de_counts
+
+    while running.get():
+
+        await asyncio.sleep(2)
+
+        print(
+            f"Graph embeddings: {de_counts:10d}"
+        )
+
+async def loader(running, de_queue, path, format, user, collection):
+
+    if format == "json":
+
+        raise RuntimeError("Not implemented")
+
+    else:
+
+        with open(path, "rb") as f:
+
+            unpacker = msgpack.Unpacker(f, raw=False)
+
+            while running.get():
+
+                try:
+                    unpacked = unpacker.unpack()
+                except:
+                    break
+
+                if user:
+                    unpacked["metadata"]["user"] = user
+
+                if collection:
+                    unpacked["metadata"]["collection"] = collection
+
+                if unpacked[0] == "de":
+                    qtype = de_queue
+
+                while running.get():
+
+                    try:
+                        await asyncio.wait_for(qtype.put(unpacked[1]), 0.5)
+
+                        # Successful put message, move on
+                        break
+
+                    except:
+                        # Hopefully it's TimeoutError.  Annoying to match since
+                        # it changed in 3.11.
+                        continue
+
+                if not running.get(): break
+
+        # Put 'None' on end of queue to finish
+        while running.get():
+
+            try:
+                await asyncio.wait_for(de_queue.put(None), 1)
+
+                # Successful put message, move on
+                break
+
+            except:
+                # Hopefully it's TimeoutError.  Annoying to match since
+                # it changed in 3.11.
+                continue
+
+async def run(running, **args):
+
+    # Maxsize on queues reduces back-pressure so tg-load-kg-core doesn't
+    # grow to eat all memory
+    de_q = asyncio.Queue(maxsize=10)
+
+    load_task = asyncio.create_task(
+        loader(
+            running=running,
+            de_queue=de_q,
+            path=args["input_file"], format=args["format"],
+            user=args["user"], collection=args["collection"],
+        )
+        
+    )
+
+    de_task = asyncio.create_task(
+        load_de(
+            running=running,
+            queue=de_q, url=args["url"] + "api/v1/"
+        )
+    )
+
+    stats_task = asyncio.create_task(stats(running))
+
+    await de_task
+
+    running.stop()
+
+    await load_task
+    await stats_task
+
+async def main(running):
+    
+    parser = argparse.ArgumentParser(
+        prog='tg-load-kg-core',
+        description=__doc__,
+    )
+
+    default_url = os.getenv("TRUSTGRAPH_API", "http://localhost:8088/")
+    default_user = "trustgraph"
+    collection = "default"
+
+    parser.add_argument(
+        '-u', '--url',
+        default=default_url,
+        help=f'TrustGraph API URL (default: {default_url})',
+    )
+
+    parser.add_argument(
+        '-i', '--input-file',
+        # Make it mandatory, difficult to over-write an existing file
+        required=True,
+        help=f'Output file'
+    )
+
+    parser.add_argument(
+        '--format',
+        default="msgpack",
+        choices=["msgpack", "json"],
+        help=f'Output format (default: msgpack)',
+    )
+
+    parser.add_argument(
+        '--user',
+        help=f'User ID to load as (default: from input)'
+    )
+
+    parser.add_argument(
+        '--collection',
+        help=f'Collection ID to load as (default: from input)'
+    )
+
+    args = parser.parse_args()
+
+    await run(running, **vars(args))
+
+running = Running()
+
+def interrupt(sig, frame):
+    running.stop()
+    print('Interrupt')
+
+signal.signal(signal.SIGINT, interrupt)
+
+asyncio.run(main(running))
+

trustgraph-cli/scripts/tg-load-kg-core

@@ -51,8 +51,13 @@ async def load_ge(running, queue, url):
                         "user": msg["m"]["u"],
                         "collection": msg["m"]["c"],
                     },
-                    "vectors": msg["v"],
+                    "entities": [
-                    "entity": msg["e"],
+                        {
+                            "entity": ent["e"],
+                            "vectors": ent["v"],
+                        }
+                        for ent in msg["e"]
+                    ],
                 }
 
                 try:

trustgraph-cli/scripts/tg-load-pdf

@@ -6,21 +6,19 @@ Loads a PDF document into TrustGraph processing.
 
 import pulsar
 from pulsar.schema import JsonSchema
-import base64
 import hashlib
 import argparse
 import os
 import time
 import uuid
 
-from trustgraph.schema import Document, document_ingest_queue
+from trustgraph.api import Api
-from trustgraph.schema import Metadata, Triple, Value
+from trustgraph.knowledge import hash, to_uri
-from trustgraph.log_level import LogLevel
-from trustgraph.knowledge import hash, to_uri, Uri
 from trustgraph.knowledge import PREF_PUBEV, PREF_DOC, PREF_ORG
 from trustgraph.knowledge import Organization, PublicationEvent
 from trustgraph.knowledge import DigitalDocument
 
+default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
 default_user = 'trustgraph'
 default_collection = 'default'
 
@@ -28,33 +26,13 @@ class Loader:
 
     def __init__(
             self,
-            pulsar_host,
+            url,
-            output_queue,
             user,
             collection,
-            log_level,
             metadata,
-            pulsar_api_key=None,
     ):
-        
+
-        if pulsar_api_key:
+        self.api = Api(url)
-            auth = pulsar.AuthenticationToken(pulsar_api_key)
-            self.client = pulsar.Client(
-				pulsar_host,
-				authentication=auth,
-				logger=pulsar.ConsoleLogger(log_level.to_pulsar())
-            )
-        else:
-            self.client = pulsar.Client(
-            pulsar_host,
-            logger=pulsar.ConsoleLogger(log_level.to_pulsar())
-            )
-            
-        self.producer = self.client.create_producer(
-            topic=output_queue,
-            schema=JsonSchema(Document),
-            chunking_enabled=True,
-        )
 
         self.user = user
         self.collection = collection
@@ -77,49 +55,18 @@ class Loader:
 
             id = to_uri(PREF_DOC, id)
 
-            triples = []
-
-            def emit(t):
-                triples.append(t)
-
             self.metadata.id = id
-            self.metadata.emit(emit)
 
-            r = Document(
+            self.api.load_document(
-                metadata=Metadata(
+                document=data, id=id, metadata=self.metadata, 
-                    id=id,
+#                user=self.user,
-                    metadata=[
+#                    collection=self.collection,
-                        Triple(
-                            s=Value(
-                                value=t["s"],
-                                is_uri=isinstance(t["s"], Uri)
-                            ),
-                            p=Value(
-                                value=t["p"],
-                                is_uri=isinstance(t["p"], Uri)
-                            ),
-                            o=Value(
-                                value=t["o"],
-                                is_uri=isinstance(t["o"], Uri)
-                            ),
-                        )
-                        for t in triples
-                    ],
-                    user=self.user,
-                    collection=self.collection,
-                ),
-                data=base64.b64encode(data),
             )
 
-            self.producer.send(r)
-
             print(f"{file}: Loaded successfully.")
 
         except Exception as e:
             print(f"{file}: Failed: {str(e)}", flush=True)
-            
-    def __del__(self):
-        self.client.close()
 
 def main():
 
@@ -128,36 +75,20 @@ def main():
         description=__doc__,
     )
 
-    default_pulsar_host = os.getenv("PULSAR_HOST", 'pulsar://localhost:6650')
-    default_pulsar_api_key = os.getenv("PULSAR_API_KEY", None)
-    default_output_queue = document_ingest_queue
-
     parser.add_argument(
-        '-p', '--pulsar-host',
+        '-u', '--url',
-        default=default_pulsar_host,
+        default=default_url,
-        help=f'Pulsar host (default: {default_pulsar_host})',
+        help=f'API URL (default: {default_url})',
     )
     
     parser.add_argument(
-        '--pulsar-api-key',
+        '-U', '--user',
-        default=default_pulsar_api_key,
-        help=f'Pulsar API key',
-    )
-
-    parser.add_argument(
-        '-o', '--output-queue',
-        default=default_output_queue,
-        help=f'Output queue (default: {default_output_queue})'
-    )
-
-    parser.add_argument(
-        '-u', '--user',
         default=default_user,
         help=f'User ID (default: {default_user})'
     )
 
     parser.add_argument(
-        '-c', '--collection',
+        '-C', '--collection',
         default=default_collection,
         help=f'Collection ID (default: {default_collection})'
     )
@@ -199,7 +130,7 @@ def main():
     )
 
     parser.add_argument(
-        '--url', help=f'Document URL'
+        '--document-url', help=f'Document URL'
     )
 
     parser.add_argument(
@@ -210,14 +141,6 @@ def main():
         '--identifier', '--id', help=f'Document ID'
     )
 
-    parser.add_argument(
-        '-l', '--log-level',
-        type=LogLevel,
-        default=LogLevel.ERROR,
-        choices=list(LogLevel),
-        help=f'Output queue (default: info)'
-    )
-
     parser.add_argument(
         'files', nargs='+',
         help=f'File to load'
@@ -237,7 +160,7 @@ def main():
                 copyright_holder=args.copyright_holder,
                 copyright_year=args.copyright_year,
                 license=args.license,
-                url=args.url,
+                url=args.document_url,
                 keywords=args.keyword,
             )
 
@@ -255,12 +178,9 @@ def main():
                 )
 
             p = Loader(
-                pulsar_host=args.pulsar_host,
+                url=args.url,
-                pulsar_api_key=args.pulsar_api_key,
-                output_queue=args.output_queue,
                 user=args.user,
                 collection=args.collection,
-                log_level=args.log_level,
                 metadata=document,
             )
 

trustgraph-cli/scripts/tg-load-text

@@ -12,14 +12,13 @@ import os
 import time
 import uuid
 
-from trustgraph.schema import TextDocument, text_ingest_queue
+from trustgraph.api import Api
-from trustgraph.schema import Metadata, Triple, Value
+from trustgraph.knowledge import hash, to_uri
-from trustgraph.log_level import LogLevel
-from trustgraph.knowledge import hash, to_uri, Literal, Uri
 from trustgraph.knowledge import PREF_PUBEV, PREF_DOC, PREF_ORG
 from trustgraph.knowledge import Organization, PublicationEvent
 from trustgraph.knowledge import DigitalDocument
 
+default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
 default_user = 'trustgraph'
 default_collection = 'default'
 
@@ -27,32 +26,14 @@ class Loader:
 
     def __init__(
             self,
-            pulsar_host,
+            url,
-            output_queue,
             user,
             collection,
-            log_level,
             metadata,
             pulsar_api_key=None,
     ):
-        if pulsar_api_key:
-            auth = pulsar.AuthenticationToken(pulsar_api_key)
-            self.client = pulsar.Client(
-                pulsar_host,
-                authentication=auth,
-                logger=pulsar.ConsoleLogger(log_level.to_pulsar())
-            )
-        else:
-            self.client = pulsar.Client(
-                pulsar_host,
-                logger=pulsar.ConsoleLogger(log_level.to_pulsar())
-            )
 
-        self.producer = self.client.create_producer(
+        self.api = Api(url)
-            topic=output_queue,
-            schema=JsonSchema(TextDocument),
-            chunking_enabled=True,
-        )
 
         self.user = user
         self.collection = collection
@@ -75,49 +56,18 @@ class Loader:
 
             id = to_uri(PREF_DOC, id)
 
-            triples = []
-
-            def emit(t):
-                triples.append(t)
-
             self.metadata.id = id
-            self.metadata.emit(emit)
 
-            r = TextDocument(
+            self.api.load_text(
-                metadata=Metadata(
+                text=data, id=id, metadata=self.metadata, 
-                    id=id,
+#                user=self.user,
-                    metadata=[
+#                    collection=self.collection,
-                        Triple(
-                            s=Value(
-                                value=t["s"],
-                                is_uri=isinstance(t["s"], Uri)
-                            ),
-                            p=Value(
-                                value=t["p"],
-                                is_uri=isinstance(t["p"], Uri)
-                            ),
-                            o=Value(
-                                value=t["o"],
-                                is_uri=isinstance(t["o"], Uri)
-                            ),
-                        )
-                        for t in triples
-                    ],
-                    user=self.user,
-                    collection=self.collection,
-                ),
-                text=data,
             )
 
-            self.producer.send(r)
-
             print(f"{file}: Loaded successfully.")
 
         except Exception as e:
             print(f"{file}: Failed: {str(e)}", flush=True)
-            
-    def __del__(self):
-        self.client.close()
 
 def main():
 
@@ -126,37 +76,20 @@ def main():
         description=__doc__,
     )
 
-    default_pulsar_host = os.getenv("PULSAR_HOST", 'pulsar://localhost:6650')
-    default_pulsar_api_key = os.getenv("PULSAR_API_KEY", None)
-
-    default_output_queue = text_ingest_queue
-
     parser.add_argument(
-        '-p', '--pulsar-host',
+        '-u', '--url',
-        default=default_pulsar_host,
+        default=default_url,
-        help=f'Pulsar host (default: {default_pulsar_host})',
+        help=f'API URL (default: {default_url})',
-    )
-    
-    parser.add_argument(
-        '--pulsar-api-key',
-        default=default_pulsar_api_key,
-        help=f'Pulsar API key',
     )
 
     parser.add_argument(
-        '-o', '--output-queue',
+        '-U', '--user',
-        default=default_output_queue,
-        help=f'Output queue (default: {default_output_queue})'
-    )
-
-    parser.add_argument(
-        '-u', '--user',
         default=default_user,
         help=f'User ID (default: {default_user})'
     )
 
     parser.add_argument(
-        '-c', '--collection',
+        '-C', '--collection',
         default=default_collection,
         help=f'Collection ID (default: {default_collection})'
     )
@@ -198,7 +131,7 @@ def main():
     )
 
     parser.add_argument(
-        '--url', help=f'Document URL'
+        '--document-url', help=f'Document URL'
     )
 
     parser.add_argument(
@@ -209,14 +142,6 @@ def main():
         '--identifier', '--id', help=f'Document ID'
     )
 
-    parser.add_argument(
-        '-l', '--log-level',
-        type=LogLevel,
-        default=LogLevel.ERROR,
-        choices=list(LogLevel),
-        help=f'Output queue (default: info)'
-    )
-
     parser.add_argument(
         'files', nargs='+',
         help=f'File to load'
@@ -236,7 +161,7 @@ def main():
                 copyright_holder=args.copyright_holder,
                 copyright_year=args.copyright_year,
                 license=args.license,
-                url=args.url,
+                url=args.document_url,
                 keywords=args.keyword,
             )
 
@@ -254,12 +179,9 @@ def main():
                 )
 
             p = Loader(
-                pulsar_host=args.pulsar_host,
+                url=args.url,
-                pulsar_api_key=args.pulsar_api_key,
-                output_queue=args.output_queue,
                 user=args.user,
                 collection=args.collection,
-                log_level=args.log_level,
                 metadata=document,
             )
 

trustgraph-cli/scripts/tg-query-document-rag

@@ -1,76 +0,0 @@
-#!/usr/bin/env python3
-
-"""
-Uses the Document RAG service to answer a query
-"""
-
-import argparse
-import os
-from trustgraph.clients.document_rag_client import DocumentRagClient
-
-default_pulsar_host = os.getenv("PULSAR_HOST", 'pulsar://localhost:6650')
-default_pulsar_api_key = os.getenv("PULSAR_API_KEY", None)
-
-default_user = 'trustgraph'
-default_collection = 'default'
-
-def query(pulsar_host, query, user, collection, pulsar_api_key=None):
-
-    rag = DocumentRagClient(pulsar_host=pulsar_host, pulsar_api_key=pulsar_api_key)
-    resp = rag.request(user=user, collection=collection, query=query)
-    print(resp)
-
-def main():
-
-    parser = argparse.ArgumentParser(
-        prog='tg-query-document-rag',
-        description=__doc__,
-    )
-
-    parser.add_argument(
-        '-p', '--pulsar-host',
-        default=default_pulsar_host,
-        help=f'Pulsar host (default: {default_pulsar_host})',
-    )
-    
-    parser.add_argument(
-        '--pulsar-api-key',
-        default=default_pulsar_api_key,
-        help=f'Pulsar API key',
-    )
-    parser.add_argument(
-        '-q', '--query',
-        required=True,
-        help=f'Query to execute',
-    )
-
-    parser.add_argument(
-        '-u', '--user',
-        default=default_user,
-        help=f'User ID (default: {default_user})'
-    )
-
-    parser.add_argument(
-        '-c', '--collection',
-        default=default_collection,
-        help=f'Collection ID (default: {default_collection})'
-    )
-
-    args = parser.parse_args()
-
-    try:
-
-        query(
-            pulsar_host=args.pulsar_host,
-            pulsar_api_key=args.pulsar_api_key,
-            query=args.query,
-            user=args.user,
-            collection=args.collection,
-        )
-
-    except Exception as e:
-
-        print("Exception:", e, flush=True)
-
-main()
-

trustgraph-cli/scripts/tg-query-graph-rag

@@ -1,77 +0,0 @@
-#!/usr/bin/env python3
-
-"""
-Uses the GraphRAG service to answer a query
-"""
-
-import argparse
-import os
-from trustgraph.clients.graph_rag_client import GraphRagClient
-
-default_pulsar_host = os.getenv("PULSAR_HOST", 'pulsar://localhost:6650')
-default_pulsar_api_key = os.getenv("PULSAR_API_KEY", None)
-
-default_user = 'trustgraph'
-default_collection = 'default'
-
-def query(pulsar_host, query, user, collection, pulsar_api_key=None):
-
-    rag = GraphRagClient(pulsar_host=pulsar_host, pulsar_api_key=pulsar_api_key)
-    resp = rag.request(user=user, collection=collection, query=query)
-    print(resp)
-
-def main():
-
-    parser = argparse.ArgumentParser(
-        prog='tg-graph-query-rag',
-        description=__doc__,
-    )
-
-    parser.add_argument(
-        '-p', '--pulsar-host',
-        default=default_pulsar_host,
-        help=f'Pulsar host (default: {default_pulsar_host})',
-    )
-
-    parser.add_argument(
-        '--pulsar-api-key',
-        default=default_pulsar_api_key,
-        help=f'Pulsar API key',
-    )
-
-    parser.add_argument(
-        '-q', '--query',
-        required=True,
-        help=f'Query to execute',
-    )
-
-    parser.add_argument(
-        '-u', '--user',
-        default=default_user,
-        help=f'User ID (default: {default_user})'
-    )
-
-    parser.add_argument(
-        '-c', '--collection',
-        default=default_collection,
-        help=f'Collection ID (default: {default_collection})'
-    )
-
-    args = parser.parse_args()
-
-    try:
-
-        query(
-            pulsar_host=args.pulsar_host,
-            pulsar_api_key=args.pulsar_api_key,
-            query=args.query,
-            user=args.user,
-            collection=args.collection,
-        )
-
-    except Exception as e:
-
-        print("Exception:", e, flush=True)
-
-main()
-

trustgraph-cli/scripts/tg-save-doc-embeds

@@ -0,0 +1,198 @@
+#!/usr/bin/env python3
+
+"""
+This utility connects to a running TrustGraph through the API and creates
+a knowledge core from the data streaming through the processing queues.
+For completeness of data, tg-save-kg-core should be initiated before data
+loading takes place.  The default output  format, msgpack should be used.
+JSON output format is also available - msgpack produces a more compact
+representation, which is also more performant to load.
+"""
+
+import aiohttp
+import asyncio
+import msgpack
+import json
+import sys
+import argparse
+import os
+import signal
+
+class Running:
+    def __init__(self): self.running = True
+    def get(self): return self.running
+    def stop(self): self.running = False
+
+async def fetch_de(running, queue, user, collection, url):
+
+    async with aiohttp.ClientSession() as session:
+
+        de_url = f"{url}stream/document-embeddings"
+
+        async with session.ws_connect(de_url) as ws:
+
+            while running.get():
+
+                try:
+                    msg = await asyncio.wait_for(ws.receive(), 1)
+                except:
+                    continue
+
+                if msg.type == aiohttp.WSMsgType.TEXT:
+
+                    data = msg.json()
+
+                    if user:
+                        if data["metadata"]["user"] != user:
+                            continue
+
+                    if collection:
+                        if data["metadata"]["collection"] != collection:
+                            continue
+
+                    await queue.put([
+                        "de",
+                        {
+                            "m": {
+                                "i": data["metadata"]["id"], 
+                                "m": data["metadata"]["metadata"],
+                                "u": data["metadata"]["user"],
+                                "c": data["metadata"]["collection"],
+                            },
+                            "c": [
+                                {
+                                    "c": chunk["chunk"],
+                                    "v": chunk["vectors"],
+                                }
+                                for chunk in data["chunks"]
+                            ]
+                        }
+                    ])
+                if msg.type == aiohttp.WSMsgType.ERROR:
+                    print("Error")
+                    break
+
+de_counts = 0
+
+async def stats(running):
+
+    global t_counts
+    global de_counts
+
+    while running.get():
+
+        await asyncio.sleep(2)
+
+        print(
+            f"Document embeddings: {de_counts:10d}"
+        )
+
+async def output(running, queue, path, format):
+
+    global t_counts
+    global de_counts
+    
+    with open(path, "wb") as f:
+
+        while running.get():
+
+            try:
+                msg = await asyncio.wait_for(queue.get(), 0.5)
+            except:
+                # Hopefully it's TimeoutError.  Annoying to match since
+                # it changed in 3.11.
+                continue
+
+            if format == "msgpack":
+                f.write(msgpack.packb(msg, use_bin_type=True))
+            else:
+                f.write(json.dumps(msg).encode("utf-8"))
+
+            if msg[0] == "de":
+                de_counts += 1
+
+    print("Output file closed")
+
+async def run(running, **args):
+
+    q = asyncio.Queue()
+
+    de_task = asyncio.create_task(
+        fetch_de(
+            running=running,
+            queue=q, user=args["user"], collection=args["collection"],
+            url=args["url"] + "api/v1/"
+        )
+    )
+
+    output_task = asyncio.create_task(
+        output(
+            running=running, queue=q,
+            path=args["output_file"], format=args["format"],
+        )
+        
+    )
+
+    stats_task = asyncio.create_task(stats(running))
+
+    await output_task
+    await de_task
+    await stats_task
+
+    print("Exiting")
+
+async def main(running):
+    
+    parser = argparse.ArgumentParser(
+        prog='tg-save-kg-core',
+        description=__doc__,
+    )
+
+    default_url = os.getenv("TRUSTGRAPH_API", "http://localhost:8088/")
+    default_user = "trustgraph"
+    collection = "default"
+
+    parser.add_argument(
+        '-u', '--url',
+        default=default_url,
+        help=f'TrustGraph API URL (default: {default_url})',
+    )
+
+    parser.add_argument(
+        '-o', '--output-file',
+        # Make it mandatory, difficult to over-write an existing file
+        required=True,
+        help=f'Output file'
+    )
+
+    parser.add_argument(
+        '--format',
+        default="msgpack",
+        choices=["msgpack", "json"],
+        help=f'Output format (default: msgpack)',
+    )
+
+    parser.add_argument(
+        '--user',
+        help=f'User ID to filter on (default: no filter)'
+    )
+
+    parser.add_argument(
+        '--collection',
+        help=f'Collection ID to filter on (default: no filter)'
+    )
+
+    args = parser.parse_args()
+
+    await run(running, **vars(args))
+
+running = Running()
+
+def interrupt(sig, frame):
+    running.stop()
+    print('Interrupt')
+
+signal.signal(signal.SIGINT, interrupt)
+
+asyncio.run(main(running))
+

trustgraph-cli/scripts/tg-save-kg-core

@@ -57,8 +57,13 @@ async def fetch_ge(running, queue, user, collection, url):
                                 "u": data["metadata"]["user"],
                                 "c": data["metadata"]["collection"],
                             },
-                            "v": data["vectors"],
+                            "e": [
-                            "e": data["entity"],
+                                {
+                                    "e": ent["entity"],
+                                    "v": ent["vectors"],
+                                }
+                                for ent in data["entities"]
+                            ]
                         }
                     ])
                 if msg.type == aiohttp.WSMsgType.ERROR:

trustgraph-cli/setup.py

@@ -34,29 +34,33 @@ setuptools.setup(
     python_requires='>=3.8',
     download_url = "https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v" + version + ".tar.gz",
     install_requires=[
-        "trustgraph-base>=0.18,<0.19",
+        "trustgraph-base>=0.21,<0.22",
         "requests",
         "pulsar-client",
+        "aiohttp",
         "rdflib",
         "tabulate",
         "msgpack",
+        "websockets",
     ],
     scripts=[
+        "scripts/tg-dump-msgpack",
         "scripts/tg-graph-show",
         "scripts/tg-graph-to-turtle",
+        "scripts/tg-init-pulsar",
         "scripts/tg-init-pulsar-manager",
+        "scripts/tg-invoke-agent",
+        "scripts/tg-invoke-document-rag",
+        "scripts/tg-invoke-graph-rag",
+        "scripts/tg-invoke-llm",
+        "scripts/tg-invoke-prompt",
+        "scripts/tg-load-kg-core",
+        "scripts/tg-load-doc-embeds",
         "scripts/tg-load-pdf",
         "scripts/tg-load-text",
         "scripts/tg-load-turtle",
-        "scripts/tg-query-document-rag",
-        "scripts/tg-query-graph-rag",
-        "scripts/tg-init-pulsar",
         "scripts/tg-processor-state",
-        "scripts/tg-invoke-agent",
-        "scripts/tg-invoke-prompt",
-        "scripts/tg-invoke-llm",
         "scripts/tg-save-kg-core",
-        "scripts/tg-load-kg-core",
+        "scripts/tg-save-doc-embeds",
-        "scripts/tg-dump-msgpack",
     ]
 )

trustgraph-embeddings-hf/setup.py

@@ -34,8 +34,8 @@ setuptools.setup(
     python_requires='>=3.8',
     download_url = "https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v" + version + ".tar.gz",
     install_requires=[
-        "trustgraph-base>=0.18,<0.19",
+        "trustgraph-base>=0.21,<0.22",
-        "trustgraph-flow>=0.18,<0.19",
+        "trustgraph-flow>=0.21,<0.22",
         "torch",
         "urllib3",
         "transformers",

trustgraph-flow/scripts/document-embeddings

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.embeddings.document_embeddings import run
+
+run()
+

trustgraph-flow/scripts/embeddings-fastembed

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.embeddings.fastembed import run
+
+run()
+

trustgraph-flow/scripts/embeddings-vectorize

@@ -1,6 +0,0 @@
-#!/usr/bin/env python3
-
-from trustgraph.embeddings.vectorize import run
-
-run()
-

trustgraph-flow/scripts/graph-embeddings

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.embeddings.graph_embeddings import run
+
+run()
+

trustgraph-flow/setup.py

@@ -34,58 +34,63 @@ setuptools.setup(
     python_requires='>=3.8',
     download_url = "https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v" + version + ".tar.gz",
     install_requires=[
-        "trustgraph-base>=0.18,<0.19",
+        "trustgraph-base>=0.21,<0.22",
-        "urllib3",
+        "aiohttp",
-        "rdflib",
-        "pymilvus",
-        "langchain",
-        "langchain-core",
-        "langchain-text-splitters",
-        "langchain-community",
-        "requests",
-        "cassandra-driver",
-        "pulsar-client",
-        "pypdf",
-        "qdrant-client",
-        "tabulate",
         "anthropic",
-        "pyyaml",
+        "cassandra-driver",
-        "prometheus-client",
         "cohere",
-        "openai",
+        "cryptography",
-        "neo4j",
+        "falkordb",
-        "tiktoken",
+        "fastembed",
         "google-generativeai",
         "ibis",
         "jsonschema",
-        "aiohttp",
+        "langchain",
+        "langchain-community",
+        "langchain-core",
+        "langchain-text-splitters",
+        "neo4j",
+        "ollama",
+        "openai",
         "pinecone[grpc]",
-        "falkordb",
+        "prometheus-client",
+        "pulsar-client",
+        "pymilvus",
+        "pypdf",
+        "pyyaml",
+        "qdrant-client",
+        "rdflib",
+        "requests",
+        "tabulate",
+        "tiktoken",
+        "urllib3",
     ],
     scripts=[
-        "scripts/api-gateway",
         "scripts/agent-manager-react",
+        "scripts/api-gateway",
         "scripts/chunker-recursive",
         "scripts/chunker-token",
         "scripts/de-query-milvus",
-        "scripts/de-query-qdrant",
         "scripts/de-query-pinecone",
+        "scripts/de-query-qdrant",
         "scripts/de-write-milvus",
-        "scripts/de-write-qdrant",
         "scripts/de-write-pinecone",
+        "scripts/de-write-qdrant",
+        "scripts/document-embeddings",
         "scripts/document-rag",
         "scripts/embeddings-ollama",
-        "scripts/embeddings-vectorize",
+        "scripts/embeddings-fastembed",
         "scripts/ge-query-milvus",
         "scripts/ge-query-pinecone",
         "scripts/ge-query-qdrant",
         "scripts/ge-write-milvus",
         "scripts/ge-write-pinecone",
         "scripts/ge-write-qdrant",
+        "scripts/graph-embeddings",
         "scripts/graph-rag",
         "scripts/kg-extract-definitions",
-        "scripts/kg-extract-topics",
         "scripts/kg-extract-relationships",
+        "scripts/kg-extract-topics",
         "scripts/metering",
         "scripts/object-extract-row",
         "scripts/oe-write-milvus",
@@ -103,13 +108,13 @@ setuptools.setup(
         "scripts/text-completion-ollama",
         "scripts/text-completion-openai",
         "scripts/triples-query-cassandra",
-        "scripts/triples-query-neo4j",
-        "scripts/triples-query-memgraph",
         "scripts/triples-query-falkordb",
+        "scripts/triples-query-memgraph",
+        "scripts/triples-query-neo4j",
         "scripts/triples-write-cassandra",
-        "scripts/triples-write-neo4j",
-        "scripts/triples-write-memgraph",
         "scripts/triples-write-falkordb",
+        "scripts/triples-write-memgraph",
+        "scripts/triples-write-neo4j",
         "scripts/wikipedia-lookup",
     ]
 )

trustgraph-flow/trustgraph/agent/react/service.py

@@ -14,8 +14,6 @@ from ... schema import AgentRequest, AgentResponse, AgentStep
 from ... schema import agent_request_queue, agent_response_queue
 from ... schema import prompt_request_queue as pr_request_queue
 from ... schema import prompt_response_queue as pr_response_queue
-from ... schema import text_completion_request_queue as tc_request_queue
-from ... schema import text_completion_response_queue as tc_response_queue
 from ... schema import graph_rag_request_queue as gr_request_queue
 from ... schema import graph_rag_response_queue as gr_response_queue
 from ... clients.prompt_client import PromptClient
@@ -133,12 +131,6 @@ class Processor(ConsumerProducer):
         prompt_response_queue = params.get(
             "prompt_response_queue", pr_response_queue
         )
-        text_completion_request_queue = params.get(
-            "text_completion_request_queue", tc_request_queue
-        )
-        text_completion_response_queue = params.get(
-            "text_completion_response_queue", tc_response_queue
-        )
         graph_rag_request_queue = params.get(
             "graph_rag_request_queue", gr_request_queue
         )
@@ -155,8 +147,6 @@ class Processor(ConsumerProducer):
                 "output_schema": AgentResponse,
                 "prompt_request_queue": prompt_request_queue,
                 "prompt_response_queue": prompt_response_queue,
-                "text_completion_request_queue": tc_request_queue,
-                "text_completion_response_queue": tc_response_queue,
                 "graph_rag_request_queue": gr_request_queue,
                 "graph_rag_response_queue": gr_response_queue,
             }
@@ -170,14 +160,6 @@ class Processor(ConsumerProducer):
             pulsar_api_key=self.pulsar_api_key,
         )
 
-        self.llm = LlmClient(
-            subscriber=subscriber,
-            input_queue=text_completion_request_queue,
-            output_queue=text_completion_response_queue,
-            pulsar_host = self.pulsar_host,
-            pulsar_api_key=self.pulsar_api_key,
-        )
-
         self.graph_rag = GraphRagClient(
             subscriber=subscriber,
             input_queue=graph_rag_request_queue,
@@ -343,18 +325,6 @@ class Processor(ConsumerProducer):
             help=f'Prompt response queue (default: {pr_response_queue})',
         )
 
-        parser.add_argument(
-            '--text-completion-request-queue',
-            default=tc_request_queue,
-            help=f'Text completion request queue (default: {tc_request_queue})',
-        )
-
-        parser.add_argument(
-            '--text-completion-response-queue',
-            default=tc_response_queue,
-            help=f'Text completion response queue (default: {tc_response_queue})',
-        )
-
         parser.add_argument(
             '--graph-rag-request-queue',
             default=gr_request_queue,

trustgraph-flow/trustgraph/agent/react/tools.py

@@ -14,6 +14,6 @@ class TextCompletionImpl:
         self.context = context
     def invoke(self, **arguments):
         return self.context.prompt.request(
-            "question", { "question": arguments.get("computation") }
+            "question", { "question": arguments.get("question") }
         )
 

trustgraph-flow/trustgraph/direct/cassandra.py

@@ -6,7 +6,7 @@ class TrustGraph:
 
     def __init__(
             self, hosts=None,
-            keyspace="trustgraph", table="default",
+            keyspace="trustgraph", table="default", username=None, password=None
     ):
 
         if hosts is None:
@@ -14,8 +14,13 @@ class TrustGraph:
 
         self.keyspace = keyspace
         self.table = table
+        self.username = username
             
-        self.cluster = Cluster(hosts)
+        if username and password:
+            auth_provider = PlainTextAuthProvider(username=username, password=password)
+            self.cluster = Cluster(hosts, auth_provider=auth_provider)
+        else:
+            self.cluster = Cluster(hosts)
         self.session = self.cluster.connect()
 
         self.init()

trustgraph-flow/trustgraph/document_rag.py

@@ -16,6 +16,44 @@ from . schema import document_embeddings_response_queue
 LABEL="http://www.w3.org/2000/01/rdf-schema#label"
 DEFINITION="http://www.w3.org/2004/02/skos/core#definition"
 
+class Query:
+
+    def __init__(self, rag, user, collection, verbose):
+        self.rag = rag
+        self.user = user
+        self.collection = collection
+        self.verbose = verbose
+
+    def get_vector(self, query):
+
+        if self.verbose:
+            print("Compute embeddings...", flush=True)
+
+        qembeds = self.rag.embeddings.request(query)
+
+        if self.verbose:
+            print("Done.", flush=True)
+
+        return qembeds
+
+    def get_docs(self, query):
+
+        vectors = self.get_vector(query)
+
+        if self.verbose:
+            print("Get entities...", flush=True)
+
+        docs = self.rag.de_client.request(
+            vectors, limit=self.rag.doc_limit
+        )
+
+        if self.verbose:
+            print("Docs:", flush=True)
+            for doc in docs:
+                print(doc, flush=True)
+
+        return docs
+
 class DocumentRag:
 
     def __init__(
@@ -56,7 +94,7 @@ class DocumentRag:
             print("Initialising...", flush=True)
 
         # FIXME: Configurable
-        self.entity_limit = 20
+        self.doc_limit = 20
 
         self.de_client = DocumentEmbeddingsClient(
             pulsar_host=pulsar_host,
@@ -85,42 +123,16 @@ class DocumentRag:
         if self.verbose:
             print("Initialised", flush=True)
 
-    def get_vector(self, query):
+    def query(self, query, user="trustgraph", collection="default"):
-
-        if self.verbose:
-            print("Compute embeddings...", flush=True)
-
-        qembeds = self.embeddings.request(query)
-
-        if self.verbose:
-            print("Done.", flush=True)
-
-        return qembeds
-
-    def get_docs(self, query):
-
-        vectors = self.get_vector(query)
-
-        if self.verbose:
-            print("Get entities...", flush=True)
-
-        docs = self.de_client.request(
-            vectors, self.entity_limit
-        )
-
-        if self.verbose:
-            print("Docs:", flush=True)
-            for doc in docs:
-                print(doc, flush=True)
-
-        return docs
-
-    def query(self, query):
 
         if self.verbose:
             print("Construct prompt...", flush=True)
 
-        docs = self.get_docs(query)
+        q = Query(
+            rag=self, user=user, collection=collection, verbose=self.verbose
+        )
+
+        docs = q.get_docs(query)
 
         if self.verbose:
             print("Invoke LLM...", flush=True)

trustgraph-flow/trustgraph/embeddings/document_embeddings/__init__.py

@@ -0,0 +1,3 @@
+
+from . embeddings import *
+

trustgraph-flow/trustgraph/embeddings/document_embeddings/__main__.py

@@ -1,5 +1,5 @@
 
-from . vectorize import run
+from . embeddings import run
 
 if __name__ == '__main__':
     run()

trustgraph-flow/trustgraph/embeddings/document_embeddings/embeddings.py

@@ -1,11 +1,13 @@
 
 """
-Vectorizer, calls the embeddings service to get embeddings for a chunk.
+Document embeddings, calls the embeddings service to get embeddings for a
-Input is text chunk, output is chunk and vectors.
+chunk of text.  Input is chunk of text plus metadata.
+Output is chunk plus embedding.
 """
 
-from ... schema import Chunk, ChunkEmbeddings
+from ... schema import Chunk, ChunkEmbeddings, DocumentEmbeddings
-from ... schema import chunk_ingest_queue, chunk_embeddings_ingest_queue
+from ... schema import chunk_ingest_queue
+from ... schema import document_embeddings_store_queue
 from ... schema import embeddings_request_queue, embeddings_response_queue
 from ... clients.embeddings_client import EmbeddingsClient
 from ... log_level import LogLevel
@@ -14,7 +16,7 @@ from ... base import ConsumerProducer
 module = ".".join(__name__.split(".")[1:-1])
 
 default_input_queue = chunk_ingest_queue
-default_output_queue = chunk_embeddings_ingest_queue
+default_output_queue = document_embeddings_store_queue
 default_subscriber = module
 
 class Processor(ConsumerProducer):
@@ -39,7 +41,7 @@ class Processor(ConsumerProducer):
                 "embeddings_response_queue": emb_response_queue,
                 "subscriber": subscriber,
                 "input_schema": Chunk,
-                "output_schema": ChunkEmbeddings,
+                "output_schema": DocumentEmbeddings,
             }
         )
 
@@ -51,31 +53,35 @@ class Processor(ConsumerProducer):
             subscriber=module + "-emb",
         )
 
-    def emit(self, metadata, chunk, vectors):
-
-        r = ChunkEmbeddings(metadata=metadata, chunk=chunk, vectors=vectors)
-        self.producer.send(r)
-
     def handle(self, msg):
 
         v = msg.value()
         print(f"Indexing {v.metadata.id}...", flush=True)
 
-        chunk = v.chunk.decode("utf-8")
-
         try:
 
-            vectors = self.embeddings.request(chunk)
+            vectors = self.embeddings.request(v.chunk)
 
-            self.emit(
+            embeds = [
+                ChunkEmbeddings(
+                    chunk=v.chunk,
+                    vectors=vectors,
+                )
+            ]
+
+            r = DocumentEmbeddings(
                 metadata=v.metadata,
-                chunk=chunk.encode("utf-8"),
+                chunks=embeds,
-                vectors=vectors
             )
 
+            self.producer.send(r)
+
         except Exception as e:
             print("Exception:", e, flush=True)
 
+            # Retry
+            raise e
+
         print("Done.", flush=True)
 
     @staticmethod

trustgraph-flow/trustgraph/embeddings/fastembed/__init__.py

@@ -0,0 +1,3 @@
+
+from . processor import *
+

trustgraph-flow/trustgraph/embeddings/fastembed/__main__.py

@@ -0,0 +1,7 @@
+#!/usr/bin/env python3
+
+from . processor import run
+
+if __name__ == '__main__':
+    run()
+