Merge pull request #327 from trustgraph-ai/release/v0.21

0.21 to master

.github/workflows/release.yaml

@@ -5,17 +5,17 @@ on:
   workflow_dispatch:
   push:
     tags:
-      - v0.18.*
+      - v*
 
 permissions:
   contents: read
 
 jobs:
 
-  deploy:
+  python-packages:
 
-    name: Build everything
+    name: Release Python packages
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     permissions:
       contents: write
       id-token: write
@@ -25,50 +25,73 @@ jobs:
     steps:
 
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
-
-      - name: Log in to Docker Hub
-        uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a
-        with:
-          username: ${{ vars.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_SECRET }}
-
-      - name: Install build dependencies
-        run: pip3 install jsonnet
 
       - name: Get version
         id: version
         run: echo VERSION=$(git describe --exact-match --tags | sed 's/^v//') >> $GITHUB_OUTPUT
 
-      - run: echo ${{ steps.version.outputs.VERSION }}
-
       - name: Build packages
         run: make packages VERSION=${{ steps.version.outputs.VERSION }}
 
       - name: Publish release distributions to PyPI
         uses: pypa/gh-action-pypi-publish@release/v1
 
-      - name: Extract metadata for container
+  deploy-container-image:
-        id: meta
-        uses: docker/metadata-action@v4
-        with:
-          images: trustgraph/trustgraph-flow
-          tags: |
-            type=ref,event=branch
-            type=ref,event=pr
-            type=semver,pattern={{version}}
-            type=semver,pattern={{major}}.{{minor}}
-            type=sha
 
-      - name: Build and push Docker image
+    name: Release container image
-        id: push
+    runs-on: ubuntu-24.04
-        uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671
+    permissions:
-        with:
+      contents: write
-          context: .
+      id-token: write
-          file: ./Containerfile
+    environment:
-          push: true
+      name: release
-          tags: ${{ steps.meta.outputs.tags }}
+
-          labels: ${{ steps.meta.outputs.labels }}
+    steps:
+
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Docker Hub token
+        run: echo ${{ secrets.DOCKER_SECRET }} > docker-token.txt
+
+      - name: Authenticate with Docker hub
+        run: make docker-hub-login
+
+      - name: Get version
+        id: version
+        run: echo VERSION=$(git describe --exact-match --tags | sed 's/^v//') >> $GITHUB_OUTPUT
+
+      - name: Put version into package manifests
+        run: make update-package-versions VERSION=${{ steps.version.outputs.VERSION }}
+
+      - name: Build containers
+        run: make container VERSION=${{ steps.version.outputs.VERSION }}
+
+      - name: Push containers
+        run: make push VERSION=${{ steps.version.outputs.VERSION }}
+
+  release-bundle:
+
+    name: Upload release bundle
+    runs-on: ubuntu-24.04
+    permissions:
+      contents: write
+      id-token: write
+    environment:
+      name: release
+
+    steps:
+
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install build dependencies
+        run: pip3 install jsonnet
+
+      - name: Get version
+        id: version
+        run: echo VERSION=$(git describe --exact-match --tags | sed 's/^v//') >> $GITHUB_OUTPUT
 
       - name: Create deploy bundle
         run: templates/generate-all deploy.zip ${{ steps.version.outputs.VERSION }}

.gitignore

@@ -9,6 +9,7 @@ trustgraph-base/trustgraph/base_version.py
 trustgraph-bedrock/trustgraph/bedrock_version.py
 trustgraph-embeddings-hf/trustgraph/embeddings_hf_version.py
 trustgraph-flow/trustgraph/flow_version.py
+trustgraph-ocr/trustgraph/ocr_version.py
 trustgraph-parquet/trustgraph/parquet_version.py
 trustgraph-vertexai/trustgraph/vertexai_version.py
 trustgraph-cli/trustgraph/

Containerfile

@@ -11,15 +11,26 @@ ENV PIP_BREAK_SYSTEM_PACKAGES=1
 RUN dnf install -y python3 python3-pip python3-wheel python3-aiohttp \
     python3-rdflib
 
-RUN pip3 install torch --index-url https://download.pytorch.org/whl/cpu
+RUN pip3 install torch==2.5.1+cpu \
+    --index-url https://download.pytorch.org/whl/cpu
 
-RUN pip3 install anthropic boto3 cohere openai google-cloud-aiplatform ollama google-generativeai \
+RUN pip3 install \
-    langchain langchain-core langchain-huggingface langchain-text-splitters \
+    anthropic boto3 cohere mistralai openai google-cloud-aiplatform \
-    langchain-community pymilvus sentence-transformers transformers \
+    ollama google-generativeai \
-    huggingface-hub pulsar-client cassandra-driver pyyaml \
+    langchain==0.3.13 langchain-core==0.3.28 langchain-huggingface==0.1.2 \
+    langchain-text-splitters==0.3.4 \
+    langchain-community==0.3.13 \
+    sentence-transformers==3.4.0 transformers==4.47.1 \
+    huggingface-hub==0.27.0 \
+    pymilvus \
+    pulsar-client==3.5.0 cassandra-driver pyyaml \
     neo4j tiktoken falkordb && \
     pip3 cache purge
 
+# Most commonly used embeddings model, just build it into the container
+# image
+RUN huggingface-cli download sentence-transformers/all-MiniLM-L6-v2
+
 # ----------------------------------------------------------------------------
 # Build a container which contains the built Python packages.  The build
 # creates a bunch of left-over cruft, a separate phase means this is only
@@ -34,6 +45,7 @@ COPY trustgraph-vertexai/ /root/build/trustgraph-vertexai/
 COPY trustgraph-bedrock/ /root/build/trustgraph-bedrock/
 COPY trustgraph-embeddings-hf/ /root/build/trustgraph-embeddings-hf/
 COPY trustgraph-cli/ /root/build/trustgraph-cli/
+COPY trustgraph-ocr/ /root/build/trustgraph-ocr/
 
 WORKDIR /root/build/
 
@@ -43,6 +55,7 @@ RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-vertexai/
 RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-bedrock/
 RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-embeddings-hf/
 RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-cli/
+RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-ocr/
 
 RUN ls /root/wheels
 
@@ -61,6 +74,7 @@ RUN \
     pip3 install /root/wheels/trustgraph_bedrock-* && \
     pip3 install /root/wheels/trustgraph_embeddings_hf-* && \
     pip3 install /root/wheels/trustgraph_cli-* && \
+    pip3 install /root/wheels/trustgraph_ocr-* && \
     pip3 cache purge && \
     rm -rf /root/wheels
 

Makefile

@@ -16,6 +16,7 @@ wheels:
 	pip3 wheel --no-deps --wheel-dir dist trustgraph-bedrock/
 	pip3 wheel --no-deps --wheel-dir dist trustgraph-embeddings-hf/
 	pip3 wheel --no-deps --wheel-dir dist trustgraph-cli/
+	pip3 wheel --no-deps --wheel-dir dist trustgraph-ocr/
 
 packages: update-package-versions
 	rm -rf dist/
@@ -26,11 +27,12 @@ packages: update-package-versions
 	cd trustgraph-bedrock && python3 setup.py sdist --dist-dir ../dist/
 	cd trustgraph-embeddings-hf && python3 setup.py sdist --dist-dir ../dist/
 	cd trustgraph-cli && python3 setup.py sdist --dist-dir ../dist/
+	cd trustgraph-ocr && python3 setup.py sdist --dist-dir ../dist/
 
 pypi-upload:
 	twine upload dist/*-${VERSION}.*
 
-CONTAINER=docker.io/trustgraph/trustgraph-flow
+CONTAINER_BASE=docker.io/trustgraph
 
 update-package-versions:
 	mkdir -p trustgraph-cli/trustgraph
@@ -41,14 +43,34 @@ update-package-versions:
 	echo __version__ = \"${VERSION}\" > trustgraph-bedrock/trustgraph/bedrock_version.py
 	echo __version__ = \"${VERSION}\" > trustgraph-embeddings-hf/trustgraph/embeddings_hf_version.py
 	echo __version__ = \"${VERSION}\" > trustgraph-cli/trustgraph/cli_version.py
+	echo __version__ = \"${VERSION}\" > trustgraph-ocr/trustgraph/ocr_version.py
 	echo __version__ = \"${VERSION}\" > trustgraph/trustgraph/trustgraph_version.py
 
 container: update-package-versions
-	${DOCKER} build -f Containerfile -t ${CONTAINER}:${VERSION} \
+	${DOCKER} build -f containers/Containerfile.base \
-	    --format docker
+	    -t ${CONTAINER_BASE}/trustgraph-base:${VERSION} .
+	${DOCKER} build -f containers/Containerfile.flow \
+	    -t ${CONTAINER_BASE}/trustgraph-flow:${VERSION} .
+	${DOCKER} build -f containers/Containerfile.bedrock \
+	    -t ${CONTAINER_BASE}/trustgraph-bedrock:${VERSION} .
+	${DOCKER} build -f containers/Containerfile.vertexai \
+	    -t ${CONTAINER_BASE}/trustgraph-vertexai:${VERSION} .
+	${DOCKER} build -f containers/Containerfile.hf \
+	    -t ${CONTAINER_BASE}/trustgraph-hf:${VERSION} .
+	${DOCKER} build -f containers/Containerfile.ocr \
+	    -t ${CONTAINER_BASE}/trustgraph-ocr:${VERSION} .
+
+container.ocr:
+	${DOCKER} build -f containers/Containerfile.ocr \
+	    -t ${CONTAINER_BASE}/trustgraph-ocr:${VERSION} .
 
 push:
-	${DOCKER} push ${CONTAINER}:${VERSION}
+	${DOCKER} push ${CONTAINER_BASE}/trustgraph-base:${VERSION}
+	${DOCKER} push ${CONTAINER_BASE}/trustgraph-flow:${VERSION}
+	${DOCKER} push ${CONTAINER_BASE}/trustgraph-bedrock:${VERSION}
+	${DOCKER} push ${CONTAINER_BASE}/trustgraph-vertexai:${VERSION}
+	${DOCKER} push ${CONTAINER_BASE}/trustgraph-hf:${VERSION}
+	${DOCKER} push ${CONTAINER_BASE}/trustgraph-ocr:${VERSION}
 
 clean:
 	rm -rf wheels/
@@ -56,13 +78,13 @@ clean:
 set-version:
 	echo '"${VERSION}"' > templates/values/version.jsonnet
 
-TEMPLATES=azure bedrock claude cohere mix llamafile ollama openai vertexai \
+TEMPLATES=azure bedrock claude cohere mix llamafile mistral ollama openai vertexai \
     openai-neo4j storage
 
 DCS=$(foreach template,${TEMPLATES},${template:%=tg-launch-%.yaml})
 
-MODELS=azure bedrock claude cohere llamafile ollama openai vertexai
+MODELS=azure bedrock claude cohere llamafile mistral ollama openai vertexai
-GRAPHS=cassandra neo4j falkordb
+GRAPHS=cassandra neo4j falkordb memgraph
 
 # tg-launch-%.yaml: templates/%.jsonnet templates/components/version.jsonnet
 # 	jsonnet -Jtemplates \
@@ -104,5 +126,5 @@ update-dcs: set-version
 
 docker-hub-login:
 	cat docker-token.txt | \
-	    docker login -u trustgraph --password-stdin registry-1.docker.io
+	    ${DOCKER} login -u trustgraph --password-stdin registry-1.docker.io
 

containers/Containerfile.base

@@ -0,0 +1,48 @@
+
+# ----------------------------------------------------------------------------
+# Build an AI container.  This does the torch install which is huge, and I
+# like to avoid re-doing this.
+# ----------------------------------------------------------------------------
+
+FROM docker.io/fedora:40 AS base
+
+ENV PIP_BREAK_SYSTEM_PACKAGES=1
+
+RUN dnf install -y python3 python3-pip python3-wheel python3-aiohttp && \
+    dnf clean all
+
+RUN pip3 install --no-cache-dir pulsar-client==3.5.0
+
+# ----------------------------------------------------------------------------
+# Build a container which contains the built Python packages.  The build
+# creates a bunch of left-over cruft, a separate phase means this is only
+# needed to support package build
+# ----------------------------------------------------------------------------
+
+FROM base AS build
+
+COPY trustgraph-base/ /root/build/trustgraph-base/
+COPY trustgraph-cli/ /root/build/trustgraph-cli/
+
+WORKDIR /root/build/
+
+RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-base/
+RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-cli/
+
+# ----------------------------------------------------------------------------
+# Finally, the target container.  Start with base and add the package.
+# ----------------------------------------------------------------------------
+
+FROM base
+
+COPY --from=build /root/wheels /root/wheels
+
+RUN \
+    pip3 install --no-cache-dir /root/wheels/trustgraph_base-* && \
+    pip3 install --no-cache-dir /root/wheels/trustgraph_cli-* && \
+    rm -rf /root/wheels
+
+WORKDIR /
+
+
+

containers/Containerfile.bedrock

@@ -0,0 +1,48 @@
+
+# ----------------------------------------------------------------------------
+# Build an AI container.  This does the torch install which is huge, and I
+# like to avoid re-doing this.
+# ----------------------------------------------------------------------------
+
+FROM docker.io/fedora:40 AS base
+
+ENV PIP_BREAK_SYSTEM_PACKAGES=1
+
+RUN dnf install -y python3 python3-pip python3-wheel python3-aiohttp \
+    python3-rdflib
+
+RUN pip3 install --no-cache-dir boto3 pulsar-client==3.5.0
+
+# ----------------------------------------------------------------------------
+# Build a container which contains the built Python packages.  The build
+# creates a bunch of left-over cruft, a separate phase means this is only
+# needed to support package build
+# ----------------------------------------------------------------------------
+
+FROM base AS build
+
+COPY trustgraph-base/ /root/build/trustgraph-base/
+COPY trustgraph-bedrock/ /root/build/trustgraph-bedrock/
+
+WORKDIR /root/build/
+
+RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-base/
+RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-bedrock/
+
+RUN ls /root/wheels
+
+# ----------------------------------------------------------------------------
+# Finally, the target container.  Start with base and add the package.
+# ----------------------------------------------------------------------------
+
+FROM base
+
+COPY --from=build /root/wheels /root/wheels
+
+RUN \
+    pip3 install --no-cache-dir /root/wheels/trustgraph_base-* && \
+    pip3 install --no-cache-dir /root/wheels/trustgraph_bedrock-* && \
+    rm -rf /root/wheels
+
+WORKDIR /
+

containers/Containerfile.flow

@@ -0,0 +1,60 @@
+
+# ----------------------------------------------------------------------------
+# Build an AI container.  This does the torch install which is huge, and I
+# like to avoid re-doing this.
+# ----------------------------------------------------------------------------
+
+FROM docker.io/fedora:40 AS base
+
+ENV PIP_BREAK_SYSTEM_PACKAGES=1
+
+RUN dnf install -y python3 python3-pip python3-wheel python3-aiohttp \
+    python3-rdflib
+
+RUN pip3 install --no-cache-dir \
+    anthropic cohere mistralai openai google-generativeai \
+    ollama \
+    langchain==0.3.13 langchain-core==0.3.28 \
+    langchain-text-splitters==0.3.4 \
+    langchain-community==0.3.13 \
+    pymilvus \
+    pulsar-client==3.5.0 cassandra-driver pyyaml \
+    neo4j tiktoken falkordb && \
+    pip3 cache purge
+
+# ----------------------------------------------------------------------------
+# Build a container which contains the built Python packages.  The build
+# creates a bunch of left-over cruft, a separate phase means this is only
+# needed to support package build
+# ----------------------------------------------------------------------------
+
+FROM base AS build
+
+COPY trustgraph-base/ /root/build/trustgraph-base/
+COPY trustgraph-flow/ /root/build/trustgraph-flow/
+COPY trustgraph-cli/ /root/build/trustgraph-cli/
+
+WORKDIR /root/build/
+
+RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-base/
+RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-flow/
+RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-cli/
+
+RUN ls /root/wheels
+
+# ----------------------------------------------------------------------------
+# Finally, the target container.  Start with base and add the package.
+# ----------------------------------------------------------------------------
+
+FROM base
+
+COPY --from=build /root/wheels /root/wheels
+
+RUN \
+    pip3 install --no-cache-dir /root/wheels/trustgraph_base-* && \
+    pip3 install --no-cache-dir /root/wheels/trustgraph_flow-* && \
+    pip3 install --no-cache-dir /root/wheels/trustgraph_cli-* && \
+    rm -rf /root/wheels
+
+WORKDIR /
+

containers/Containerfile.hf

@@ -0,0 +1,75 @@
+
+# ----------------------------------------------------------------------------
+# Build an AI container.  This does the torch install which is huge, and I
+# like to avoid re-doing this.
+# ----------------------------------------------------------------------------
+
+FROM docker.io/fedora:40 AS ai
+
+ENV PIP_BREAK_SYSTEM_PACKAGES=1
+
+RUN dnf install -y python3 python3-pip python3-wheel python3-aiohttp \
+    python3-rdflib
+
+RUN pip3 install torch==2.5.1+cpu \
+    --index-url https://download.pytorch.org/whl/cpu
+
+RUN pip3 install --no-cache-dir \
+    langchain==0.3.13 langchain-core==0.3.28 langchain-huggingface==0.1.2 \
+    langchain-community==0.3.13 \
+    sentence-transformers==3.4.0 transformers==4.47.1 \
+    huggingface-hub==0.27.0 \
+    pulsar-client==3.5.0
+
+# Most commonly used embeddings model, just build it into the container
+# image
+RUN huggingface-cli download sentence-transformers/all-MiniLM-L6-v2
+
+# ----------------------------------------------------------------------------
+# Build a container which contains the built Python packages.  The build
+# creates a bunch of left-over cruft, a separate phase means this is only
+# needed to support package build
+# ----------------------------------------------------------------------------
+
+FROM ai AS build
+
+COPY trustgraph-base/ /root/build/trustgraph-base/
+COPY trustgraph-flow/ /root/build/trustgraph-flow/
+COPY trustgraph-vertexai/ /root/build/trustgraph-vertexai/
+COPY trustgraph-bedrock/ /root/build/trustgraph-bedrock/
+COPY trustgraph-embeddings-hf/ /root/build/trustgraph-embeddings-hf/
+COPY trustgraph-cli/ /root/build/trustgraph-cli/
+
+WORKDIR /root/build/
+
+RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-base/
+RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-flow/
+RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-vertexai/
+RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-bedrock/
+RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-embeddings-hf/
+RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-cli/
+
+RUN ls /root/wheels
+
+# ----------------------------------------------------------------------------
+# Finally, the target container.  Start with base and add the package.
+# ----------------------------------------------------------------------------
+
+FROM ai
+
+COPY --from=build /root/wheels /root/wheels
+
+RUN \
+    pip3 install /root/wheels/trustgraph_base-* && \
+    pip3 install /root/wheels/trustgraph_flow-* && \
+    pip3 install /root/wheels/trustgraph_vertexai-* && \
+    pip3 install /root/wheels/trustgraph_bedrock-* && \
+    pip3 install /root/wheels/trustgraph_embeddings_hf-* && \
+    pip3 install /root/wheels/trustgraph_cli-* && \
+    pip3 cache purge && \
+    rm -rf /root/wheels
+
+WORKDIR /
+
+CMD sleep 1000000
+

containers/Containerfile.ocr

@@ -0,0 +1,48 @@
+
+# ----------------------------------------------------------------------------
+# Build an AI container.  This does the torch install which is huge, and I
+# like to avoid re-doing this.
+# ----------------------------------------------------------------------------
+
+FROM docker.io/fedora:40 AS base
+
+ENV PIP_BREAK_SYSTEM_PACKAGES=1
+
+RUN dnf install -y python3 python3-pip python3-wheel python3-aiohttp \
+    python3-rdflib tesseract poppler poppler-utils
+
+RUN pip3 install --no-cache-dir pytesseract pulsar-client==3.5.0
+
+# ----------------------------------------------------------------------------
+# Build a container which contains the built Python packages.  The build
+# creates a bunch of left-over cruft, a separate phase means this is only
+# needed to support package build
+# ----------------------------------------------------------------------------
+
+FROM base AS build
+
+COPY trustgraph-base/ /root/build/trustgraph-base/
+COPY trustgraph-ocr/ /root/build/trustgraph-ocr/
+
+WORKDIR /root/build/
+
+RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-base/
+RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-ocr/
+
+RUN ls /root/wheels
+
+# ----------------------------------------------------------------------------
+# Finally, the target container.  Start with base and add the package.
+# ----------------------------------------------------------------------------
+
+FROM base
+
+COPY --from=build /root/wheels /root/wheels
+
+RUN \
+    pip3 install --no-cache-dir /root/wheels/trustgraph_base-* && \
+    pip3 install --no-cache-dir /root/wheels/trustgraph_ocr-* && \
+    rm -rf /root/wheels
+
+WORKDIR /
+

containers/Containerfile.vertexai

@@ -0,0 +1,51 @@
+
+# ----------------------------------------------------------------------------
+# Build an AI container.  This does the torch install which is huge, and I
+# like to avoid re-doing this.
+# ----------------------------------------------------------------------------
+
+FROM docker.io/fedora:40 AS base
+
+ENV PIP_BREAK_SYSTEM_PACKAGES=1
+
+RUN dnf install -y python3 python3-pip python3-wheel python3-aiohttp \
+    python3-rdflib
+
+RUN pip3 install --no-cache-dir \
+    google-cloud-aiplatform pulsar-client==3.5.0
+
+# ----------------------------------------------------------------------------
+# Build a container which contains the built Python packages.  The build
+# creates a bunch of left-over cruft, a separate phase means this is only
+# needed to support package build
+# ----------------------------------------------------------------------------
+
+FROM base AS build
+
+COPY trustgraph-base/ /root/build/trustgraph-base/
+COPY trustgraph-vertexai/ /root/build/trustgraph-vertexai/
+
+WORKDIR /root/build/
+
+RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-base/
+RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-vertexai/
+
+RUN ls /root/wheels
+
+# ----------------------------------------------------------------------------
+# Finally, the target container.  Start with base and add the package.
+# ----------------------------------------------------------------------------
+
+FROM base
+
+COPY --from=build /root/wheels /root/wheels
+
+RUN \
+    pip3 install --no-cache-dir /root/wheels/trustgraph_base-* && \
+    pip3 install --no-cache-dir /root/wheels/trustgraph_vertexai-* && \
+    rm -rf /root/wheels
+
+WORKDIR /
+
+
+

docs/README.quickstart-docker-compose.md

@@ -113,8 +113,9 @@ Choose one of the `Docker Compose` files that meets your preferred model and gra
 ### AWS Bedrock API
 
 ```
-export AWS_ID_KEY=<ID-KEY-HERE>
+export AWS_ACCESS_KEY_ID=<ID-KEY-HERE>
-export AWS_SECRET_KEY=<TOKEN-GOES-HERE>
+export AWS_SECRET_ACCESS_KEY=<TOKEN-GOES-HERE>
+export AWS_DEFAULT_REGION=<REGION-HERE>
 docker compose -f tg-launch-bedrock-cassandra.yaml up -d # Using Cassandra as the graph store
 docker compose -f tg-launch-bedrock-neo4j.yaml up -d # Using Neo4j as the graph store
 ```

grafana/dashboards/dashboard.json

@@ -577,7 +577,7 @@
           "disableTextWrap": false,
           "editorMode": "builder",
           "exemplar": false,
-          "expr": "increase(processing_count_total{status!=\"success\"}[$__rate_interval])",
+          "expr": "sum by(job) (increase(rate_limit_count_total[$__rate_interval]))",
           "format": "time_series",
           "fullMetaSearch": false,
           "includeNullMetadata": true,
@@ -588,7 +588,7 @@
           "useBackend": false
         }
       ],
-      "title": "Errors",
+      "title": "Rate limit events",
       "type": "timeseries"
     },
     {

prometheus/prometheus.yml

@@ -20,6 +20,18 @@ scrape_configs:
       - targets:
         - 'pulsar:8080'
 
+  - job_name: 'bookie'
+    scrape_interval: 5s
+    static_configs:
+      - targets:
+        - 'bookie:8000'
+
+  - job_name: 'zookeeper'
+    scrape_interval: 5s
+    static_configs:
+      - targets:
+        - 'zookeeper:8000'
+
   - job_name: 'pdf-decoder'
     scrape_interval: 5s
     static_configs:
@@ -32,11 +44,17 @@ scrape_configs:
       - targets:
         - 'chunker:8000'
 
-  - job_name: 'vectorize'
+  - job_name: 'document-embeddings'
     scrape_interval: 5s
     static_configs:
       - targets:
-        - 'vectorize:8000'
+        - 'document-embeddings:8000'
+
+  - job_name: 'graph-embeddings'
+    scrape_interval: 5s
+    static_configs:
+      - targets:
+        - 'graph-embeddings:8000'
 
   - job_name: 'embeddings'
     scrape_interval: 5s
@@ -74,6 +92,12 @@ scrape_configs:
       - targets:
         - 'metering-rag:8000'
 
+  - job_name: 'store-doc-embeddings'
+    scrape_interval: 5s
+    static_configs:
+      - targets:
+        - 'store-doc-embeddings:8000'
+
   - job_name: 'store-graph-embeddings'
     scrape_interval: 5s
     static_configs:
@@ -104,6 +128,12 @@ scrape_configs:
       - targets:
         - 'graph-rag:8000'
 
+  - job_name: 'document-rag'
+    scrape_interval: 5s
+    static_configs:
+      - targets:
+        - 'document-rag:8000'
+
   - job_name: 'prompt'
     scrape_interval: 5s
     static_configs:
@@ -122,6 +152,12 @@ scrape_configs:
       - targets:
         - 'query-graph-embeddings:8000'
 
+  - job_name: 'query-doc-embeddings'
+    scrape_interval: 5s
+    static_configs:
+      - targets:
+        - 'query-doc-embeddings:8000'
+
   - job_name: 'query-triples'
     scrape_interval: 5s
     static_configs:
@@ -145,3 +181,7 @@ scrape_configs:
     static_configs:
       - targets:
         - 'workbench-ui:8000'
+
+# Cassandra
+# qdrant
+

templates/all-patterns.jsonnet

@@ -13,6 +13,7 @@
     import "patterns/llm-claude.jsonnet",
     import "patterns/llm-cohere.jsonnet",
     import "patterns/llm-llamafile.jsonnet",
+    import "patterns/llm-mistral.jsonnet",
     import "patterns/llm-ollama.jsonnet",
     import "patterns/llm-openai.jsonnet",
     import "patterns/llm-vertexai.jsonnet",

templates/components.jsonnet

@@ -1,47 +1,80 @@
 {
+
+   // Essentials
+   "trustgraph-base": import "components/trustgraph.jsonnet",
+   "pulsar": import "components/pulsar.jsonnet",
+
+   // LLMs
    "azure": import "components/azure.jsonnet",
    "azure-openai": import "components/azure-openai.jsonnet",
    "bedrock": import "components/bedrock.jsonnet",
    "claude": import "components/claude.jsonnet",
    "cohere": import "components/cohere.jsonnet",
-   "document-rag": import "components/document-rag.jsonnet",
-   "embeddings-hf": import "components/embeddings-hf.jsonnet",
-   "embeddings-ollama": import "components/embeddings-ollama.jsonnet",
    "googleaistudio": import "components/googleaistudio.jsonnet",
-   "grafana": import "components/grafana.jsonnet",
+   "mistral": import "components/mistral.jsonnet",
+   "ollama": import "components/ollama.jsonnet",
+   "openai": import "components/openai.jsonnet",
+   "vertexai": import "components/vertexai.jsonnet",
+
+   // LLMs for RAG
+   "azure-rag": import "components/azure-rag.jsonnet",
+   "azure-openai-rag": import "components/azure-openai-rag.jsonnet",
+   "bedrock-rag": import "components/bedrock-rag.jsonnet",
+   "claude-rag": import "components/claude-rag.jsonnet",
+   "cohere-rag": import "components/cohere-rag.jsonnet",
+   "googleaistudio-rag": import "components/googleaistudio-rag.jsonnet",
+   "mistral-rag": import "components/mistral-rag.jsonnet",
+   "ollama-rag": import "components/ollama-rag.jsonnet",
+   "openai-rag": import "components/openai-rag.jsonnet",
+   "vertexai-rag": import "components/vertexai-rag.jsonnet",
+
+   // Embeddings
+   "embeddings-ollama": import "components/embeddings-ollama.jsonnet",
+   "embeddings-hf": import "components/embeddings-hf.jsonnet",
+   "embeddings-fastembed": import "components/embeddings-fastembed.jsonnet",
+
+   // Processing pipelines
    "graph-rag": import "components/graph-rag.jsonnet",
+   "document-rag": import "components/document-rag.jsonnet",
+
+   // OCR options
+   "ocr": import "components/ocr.jsonnet",
+   "mistral-ocr": import "components/mistral-ocr.jsonnet",
+
+   // Librarian - document management
+   "librarian": import "components/librarian.jsonnet",
+
+   // Vector stores
+   "vector-store-milvus": import "components/milvus.jsonnet",
+   "vector-store-qdrant": import "components/qdrant.jsonnet",
+   "vector-store-pinecone": import "components/pinecone.jsonnet",
+
+   // Triples stores
    "triple-store-cassandra": import "components/cassandra.jsonnet",
    "triple-store-neo4j": import "components/neo4j.jsonnet",
    "triple-store-falkordb": import "components/falkordb.jsonnet",
    "triple-store-memgraph": import "components/memgraph.jsonnet",
+
+   // Observability support
+   "grafana": import "components/grafana.jsonnet",
+
+   // Pulsar manager is a UI for Pulsar.  Uses a LOT of memory
+   "pulsar-manager": import "components/pulsar-manager.jsonnet",
+
    "llamafile": import "components/llamafile.jsonnet",
-   "ollama": import "components/ollama.jsonnet",
-   "openai": import "components/openai.jsonnet",
    "override-recursive-chunker": import "components/chunker-recursive.jsonnet",
 
+   // The prompt manager
    "prompt-template": import "components/prompt-template.jsonnet",
    "prompt-overrides": import "components/prompt-overrides.jsonnet",
 
-   "pulsar": import "components/pulsar.jsonnet",
+   // ReAct agent
-   "pulsar-manager": import "components/pulsar-manager.jsonnet",
-   "trustgraph-base": import "components/trustgraph.jsonnet",
-   "vector-store-milvus": import "components/milvus.jsonnet",
-   "vector-store-qdrant": import "components/qdrant.jsonnet",
-   "vector-store-pinecone": import "components/pinecone.jsonnet",
-   "vertexai": import "components/vertexai.jsonnet",
-   "workbench-ui": import "components/workbench-ui.jsonnet",
-   "null": {},
-
    "agent-manager-react": import "components/agent-manager-react.jsonnet",
 
-   // FIXME: Dupes
+   // Optional UI
-   "cassandra": import "components/cassandra.jsonnet",
+   "workbench-ui": import "components/workbench-ui.jsonnet",
-   "neo4j": import "components/neo4j.jsonnet",
+
-   "memgraph": import "components/memgraph.jsonnet",
+   // Does nothing.  But, can be a hack to overwrite parameters
-   "qdrant": import "components/qdrant.jsonnet",
+   "null": {},
-   "pinecone": import "components/pinecone.jsonnet",
-   "milvus": import "components/milvus.jsonnet",
-   "falkordb": import "components/falkordb.jsonnet",
-   "trustgraph": import "components/trustgraph.jsonnet",
 
 }

templates/components/agent-manager-react.jsonnet

@@ -14,11 +14,15 @@ local default_prompts = import "prompts/default-prompts.jsonnet";
 
             local container =
                 engine.container("agent-manager")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "agent-manager-react",
                         "-p",
                         url.pulsar,
+                        "--prompt-request-queue",
+                        "non-persistent://tg/request/prompt-rag",
+                        "--prompt-response-queue",
+                        "non-persistent://tg/response/prompt-rag",
                         "--tool-type",
                     ] + [
                         tool.id + "=" + tool.type

templates/components/azure-openai-rag.jsonnet

@@ -0,0 +1,61 @@
+local base = import "base/base.jsonnet";
+local images = import "values/images.jsonnet";
+local url = import "values/url.jsonnet";
+local prompts = import "prompts/mixtral.jsonnet";
+
+{
+
+    with:: function(key, value)
+        self + {
+            ["ollama-rag-" + key]:: value,
+        },
+
+    "azure-openai-rag-model":: "GPT-3.5-Turbo",
+    "azure-openai-rag-max-output-tokens":: 4192,
+    "azure-openai-rag-temperature":: 0.0,
+
+    "text-completion-rag" +: {
+    
+        create:: function(engine)
+
+            local envSecrets = engine.envSecrets("azure-openai-credentials")
+                .with_env_var("AZURE_TOKEN", "azure-token");
+
+            local containerRag =
+                engine.container("text-completion-rag")
+                    .with_image(images.trustgraph_flow)
+                    .with_command([
+                        "text-completion-azure",
+                        "-p",
+                        url.pulsar,
+                        "-x",
+                        std.toString($["azure-openai-rag-max-output-tokens"]),
+                        "-t",
+                        "%0.3f" % $["azure-openai-rag-temperature"],
+                        "-i",
+                        "non-persistent://tg/request/text-completion-rag",
+                        "-o",
+                        "non-persistent://tg/response/text-completion-rag",
+                    ])
+                    .with_env_var_secrets(envSecrets)
+                    .with_limits("0.5", "128M")
+                    .with_reservations("0.1", "128M");
+
+            local containerSetRag = engine.containers(
+                "text-completion-rag", [ containerRag ]
+            );
+
+            local serviceRag =
+                engine.internalService(containerSetRag)
+                .with_port(8000, 8000, "metrics");
+
+            engine.resources([
+                envSecrets,
+                containerSetRag,
+                serviceRag,
+            ])
+
+    },
+
+} + prompts
+

templates/components/azure-openai.jsonnet

@@ -5,6 +5,11 @@ local prompts = import "prompts/mixtral.jsonnet";
 
 {
 
+    with:: function(key, value)
+        self + {
+            ["azure-openai-" + key]:: value,
+        },
+
     "azure-openai-model":: "GPT-3.5-Turbo",
     "azure-openai-max-output-tokens":: 4192,
     "azure-openai-temperature":: 0.0,
@@ -18,7 +23,7 @@ local prompts = import "prompts/mixtral.jsonnet";
 
             local container =
                 engine.container("text-completion")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "text-completion-azure-openai",
                         "-p",
@@ -34,48 +39,18 @@ local prompts = import "prompts/mixtral.jsonnet";
                     .with_limits("0.5", "128M")
                     .with_reservations("0.1", "128M");
 
-            local containerRag =
-                engine.container("text-completion-rag")
-                    .with_image(images.trustgraph)
-                    .with_command([
-                        "text-completion-azure",
-                        "-p",
-                        url.pulsar,
-                        "-x",
-                        std.toString($["azure-openai-max-output-tokens"]),
-                        "-t",
-                        "%0.3f" % $["azure-openai-temperature"],
-                        "-i",
-                        "non-persistent://tg/request/text-completion-rag",
-                        "-o",
-                        "non-persistent://tg/response/text-completion-rag",
-                    ])
-                    .with_env_var_secrets(envSecrets)
-                    .with_limits("0.5", "128M")
-                    .with_reservations("0.1", "128M");
-
             local containerSet = engine.containers(
                 "text-completion", [ container ]
             );
 
-            local containerSetRag = engine.containers(
-                "text-completion-rag", [ containerRag ]
-            );
-
             local service =
                 engine.internalService(containerSet)
                 .with_port(8000, 8000, "metrics");
 
-            local serviceRag =
-                engine.internalService(containerSetRag)
-                .with_port(8000, 8000, "metrics");
-
             engine.resources([
                 envSecrets,
                 containerSet,
-                containerSetRag,
                 service,
-                serviceRag,
             ])
 
     },

templates/components/azure-rag.jsonnet

@@ -0,0 +1,60 @@
+local images = import "values/images.jsonnet";
+local url = import "values/url.jsonnet";
+local prompts = import "prompts/mixtral.jsonnet";
+
+{
+
+    with:: function(key, value)
+        self + {
+            ["azure-rag-" + key]:: value,
+        },
+
+    "azure-rag-max-output-tokens":: 4096,
+    "azure-rag-temperature":: 0.0,
+
+    "text-completion-rag" +: {
+    
+        create:: function(engine)
+
+            local envSecrets = engine.envSecrets("azure-credentials")
+                .with_env_var("AZURE_TOKEN", "azure-token")
+                .with_env_var("AZURE_ENDPOINT", "azure-endpoint");
+
+            local containerRag =
+                engine.container("text-completion-rag")
+                    .with_image(images.trustgraph_flow)
+                    .with_command([
+                        "text-completion-azure",
+                        "-p",
+                        url.pulsar,
+                        "-x",
+                        std.toString($["azure-rag-max-output-tokens"]),
+                        "-t",
+                        "%0.3f" % $["azure-rag-temperature"],
+                        "-i",
+                        "non-persistent://tg/request/text-completion-rag",
+                        "-o",
+                        "non-persistent://tg/response/text-completion-rag",
+                    ])
+                    .with_env_var_secrets(envSecrets)
+                    .with_limits("0.5", "128M")
+                    .with_reservations("0.1", "128M");
+
+            local containerSetRag = engine.containers(
+                "text-completion-rag", [ containerRag ]
+            );
+
+            local serviceRag =
+                engine.internalService(containerSetRag)
+                .with_port(8000, 8000, "metrics");
+
+            engine.resources([
+                envSecrets,
+                containerSetRag,
+                serviceRag,
+            ])
+
+    }
+
+} + prompts
+

templates/components/azure.jsonnet

@@ -1,10 +1,14 @@
-local base = import "base/base.jsonnet";
 local images = import "values/images.jsonnet";
 local url = import "values/url.jsonnet";
 local prompts = import "prompts/mixtral.jsonnet";
 
 {
 
+    with:: function(key, value)
+        self + {
+            ["azure-" + key]:: value,
+        },
+
     "azure-max-output-tokens":: 4096,
     "azure-temperature":: 0.0,
 
@@ -18,7 +22,7 @@ local prompts = import "prompts/mixtral.jsonnet";
 
             local container =
                 engine.container("text-completion")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "text-completion-azure",
                         "-p",
@@ -32,48 +36,18 @@ local prompts = import "prompts/mixtral.jsonnet";
                     .with_limits("0.5", "128M")
                     .with_reservations("0.1", "128M");
 
-            local containerRag =
-                engine.container("text-completion-rag")
-                    .with_image(images.trustgraph)
-                    .with_command([
-                        "text-completion-azure",
-                        "-p",
-                        url.pulsar,
-                        "-x",
-                        std.toString($["azure-max-output-tokens"]),
-                        "-t",
-                        "%0.3f" % $["azure-temperature"],
-                        "-i",
-                        "non-persistent://tg/request/text-completion-rag",
-                        "-o",
-                        "non-persistent://tg/response/text-completion-rag",
-                    ])
-                    .with_env_var_secrets(envSecrets)
-                    .with_limits("0.5", "128M")
-                    .with_reservations("0.1", "128M");
-
             local containerSet = engine.containers(
                 "text-completion", [ container ]
             );
 
-            local containerSetRag = engine.containers(
-                "text-completion-rag", [ containerRag ]
-            );
-
             local service =
                 engine.internalService(containerSet)
                 .with_port(8000, 8000, "metrics");
 
-            local serviceRag =
-                engine.internalService(containerSetRag)
-                .with_port(8000, 8000, "metrics");
-
             engine.resources([
                 envSecrets,
                 containerSet,
-                containerSetRag,
                 service,
-                serviceRag,
             ])
 
     }

templates/components/bedrock-rag.jsonnet

@@ -0,0 +1,66 @@
+local base = import "base/base.jsonnet";
+local images = import "values/images.jsonnet";
+local url = import "values/url.jsonnet";
+local prompts = import "prompts/mixtral.jsonnet";
+local chunker = import "chunker-recursive.jsonnet";
+
+{
+
+    with:: function(key, value)
+        self + {
+            ["bedrock-rag-" + key]:: value,
+        },
+
+    "bedrock-rag-max-output-tokens":: 4096,
+    "bedrock-rag-temperature":: 0.0,
+    "bedrock-rag-model":: "mistral.mixtral-8x7b-instruct-v0:1",
+
+    "text-completion-rag" +: {
+    
+        create:: function(engine)
+
+            local envSecrets = engine.envSecrets("bedrock-credentials")
+                .with_env_var("AWS_ACCESS_KEY_ID", "aws-id-key")
+                .with_env_var("AWS_SECRET_ACCESS_KEY", "aws-secret")
+                .with_env_var("AWS_DEFAULT_REGION", "aws-region");
+
+            local containerRag =
+                engine.container("text-completion-rag")
+                    .with_image(images.trustgraph_bedrock)
+                    .with_command([
+                        "text-completion-bedrock",
+                        "-p",
+                        url.pulsar,
+                        "-x",
+                        std.toString($["bedrock-rag-max-output-tokens"]),
+                        "-t",
+                        "%0.3f" % $["bedrock-rag-temperature"],
+                        "-m",
+                        $["bedrock-rag-model"],
+                        "-i",
+                        "non-persistent://tg/request/text-completion-rag",
+                        "-o",
+                        "non-persistent://tg/response/text-completion-rag",
+              	    ])
+                    .with_env_var_secrets(envSecrets)
+                    .with_limits("0.5", "128M")
+                    .with_reservations("0.1", "128M");
+
+            local containerSetRag = engine.containers(
+                "text-completion-rag", [ containerRag ]
+            );
+
+            local serviceRag =
+                engine.internalService(containerSetRag)
+                .with_port(8000, 8000, "metrics");
+
+            engine.resources([
+                envSecrets,
+                containerSetRag,
+                serviceRag,
+            ])
+
+    },
+
+} + prompts + chunker
+

templates/components/bedrock.jsonnet

@@ -6,6 +6,11 @@ local chunker = import "chunker-recursive.jsonnet";
 
 {
 
+    with:: function(key, value)
+        self + {
+            ["bedrock-" + key]:: value,
+        },
+
     "bedrock-max-output-tokens":: 4096,
     "bedrock-temperature":: 0.0,
     "bedrock-model":: "mistral.mixtral-8x7b-instruct-v0:1",
@@ -15,13 +20,13 @@ local chunker = import "chunker-recursive.jsonnet";
         create:: function(engine)
 
             local envSecrets = engine.envSecrets("bedrock-credentials")
-                .with_env_var("AWS_ID_KEY", "aws-id-key")
+                .with_env_var("AWS_ACCESS_KEY_ID", "aws-id-key")
-                .with_env_var("AWS_SECRET", "aws-secret")
+                .with_env_var("AWS_SECRET_ACCESS_KEY", "aws-secret")
-                .with_env_var("AWS_REGION", "aws-region");
+                .with_env_var("AWS_DEFAULT_REGION", "aws-region");
 
             local container =
                 engine.container("text-completion")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_bedrock)
                     .with_command([
                         "text-completion-bedrock",
                         "-p",
@@ -37,50 +42,18 @@ local chunker = import "chunker-recursive.jsonnet";
                     .with_limits("0.5", "128M")
                     .with_reservations("0.1", "128M");
 
-            local containerRag =
-                engine.container("text-completion-rag")
-                    .with_image(images.trustgraph)
-                    .with_command([
-                        "text-completion-bedrock",
-                        "-p",
-                        url.pulsar,
-                        "-x",
-                        std.toString($["bedrock-max-output-tokens"]),
-                        "-t",
-                        "%0.3f" % $["bedrock-temperature"],
-                        "-m",
-                        $["bedrock-model"],
-                        "-i",
-                        "non-persistent://tg/request/text-completion-rag",
-                        "-o",
-                        "non-persistent://tg/response/text-completion-rag",
-              	    ])
-                    .with_env_var_secrets(envSecrets)
-                    .with_limits("0.5", "128M")
-                    .with_reservations("0.1", "128M");
-
             local containerSet = engine.containers(
                 "text-completion", [ container ]
             );
 
-            local containerSetRag = engine.containers(
-                "text-completion-rag", [ containerRag ]
-            );
-
             local service =
                 engine.internalService(containerSet)
                 .with_port(8000, 8000, "metrics");
 
-            local serviceRag =
-                engine.internalService(containerSetRag)
-                .with_port(8000, 8000, "metrics");
-
             engine.resources([
                 envSecrets,
                 containerSet,
-                containerSetRag,
                 service,
-                serviceRag,
             ])
 
     },

templates/components/cassandra.jsonnet

@@ -12,7 +12,7 @@ cassandra + {
 
             local container =
                 engine.container("store-triples")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "triples-write-cassandra",
                         "-p",
@@ -44,7 +44,7 @@ cassandra + {
 
             local container =
                 engine.container("query-triples")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "triples-query-cassandra",
                         "-p",

templates/components/chunker-recursive.jsonnet

@@ -14,7 +14,7 @@ local prompts = import "prompts/mixtral.jsonnet";
 
             local container =
                 engine.container("chunker")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "chunker-recursive",
                         "-p",

templates/components/claude-rag.jsonnet

@@ -0,0 +1,63 @@
+local base = import "base/base.jsonnet";
+local images = import "values/images.jsonnet";
+local url = import "values/url.jsonnet";
+local prompts = import "prompts/mixtral.jsonnet";
+
+{
+
+    with:: function(key, value)
+        self + {
+            ["claude-rag-" + key]:: value,
+        },
+
+    "claude-rag-model":: "claude-3-sonnet-20240229",
+    "claude-rag-max-output-tokens":: 4096,
+    "claude-rag-temperature":: 0.0,
+
+    "text-completion-rag" +: {
+    
+        create:: function(engine)
+
+            local envSecrets = engine.envSecrets("claude-credentials")
+                .with_env_var("CLAUDE_KEY", "claude-key");
+
+            local containerRag =
+                engine.container("text-completion-rag")
+                    .with_image(images.trustgraph_flow)
+                    .with_command([
+                        "text-completion-claude",
+                        "-p",
+                        url.pulsar,
+                        "-x",
+                        std.toString($["claude-rag-max-output-tokens"]),
+                        "-m",
+                        $["claude-rag-model"],
+                        "-t",
+                        "%0.3f" % $["claude-rag-temperature"],
+                        "-i",
+                        "non-persistent://tg/request/text-completion-rag",
+                        "-o",
+                        "non-persistent://tg/response/text-completion-rag",
+                    ])
+                    .with_env_var_secrets(envSecrets)
+                    .with_limits("0.5", "128M")
+                    .with_reservations("0.1", "128M");
+
+            local containerSetRag = engine.containers(
+                "text-completion-rag", [ containerRag ]
+            );
+
+            local serviceRag =
+                engine.internalService(containerSetRag)
+                .with_port(8000, 8000, "metrics");
+
+            engine.resources([
+                envSecrets,
+                containerSetRag,
+                serviceRag,
+            ])
+
+    },
+
+} + prompts
+

templates/components/claude.jsonnet

@@ -5,6 +5,12 @@ local prompts = import "prompts/mixtral.jsonnet";
 
 {
 
+    with:: function(key, value)
+        self + {
+            ["claude-" + key]:: value,
+        },
+
+    "claude-model":: "claude-3-sonnet-20240229",
     "claude-max-output-tokens":: 4096,
     "claude-temperature":: 0.0,
 
@@ -13,17 +19,19 @@ local prompts = import "prompts/mixtral.jsonnet";
         create:: function(engine)
 
             local envSecrets = engine.envSecrets("claude-credentials")
-                .with_env_var("CLAUDE_KEY_TOKEN", "claude-key");
+                .with_env_var("CLAUDE_KEY", "claude-key");
 
             local container =
                 engine.container("text-completion")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "text-completion-claude",
                         "-p",
                         url.pulsar,
                         "-x",
                         std.toString($["claude-max-output-tokens"]),
+                        "-m",
+                        $["claude-model"],
                         "-t",
                         "%0.3f" % $["claude-temperature"],
                     ])
@@ -31,48 +39,18 @@ local prompts = import "prompts/mixtral.jsonnet";
                     .with_limits("0.5", "128M")
                     .with_reservations("0.1", "128M");
 
-            local containerRag =
-                engine.container("text-completion-rag")
-                    .with_image(images.trustgraph)
-                    .with_command([
-                        "text-completion-claude",
-                        "-p",
-                        url.pulsar,
-                        "-x",
-                        std.toString($["claude-max-output-tokens"]),
-                        "-t",
-                        "%0.3f" % $["claude-temperature"],
-                        "-i",
-                        "non-persistent://tg/request/text-completion-rag",
-                        "-o",
-                        "non-persistent://tg/response/text-completion-rag",
-                    ])
-                    .with_env_var_secrets(envSecrets)
-                    .with_limits("0.5", "128M")
-                    .with_reservations("0.1", "128M");
-
             local containerSet = engine.containers(
                 "text-completion", [ container ]
             );
 
-            local containerSetRag = engine.containers(
-                "text-completion-rag", [ containerRag ]
-            );
-
             local service =
                 engine.internalService(containerSet)
                 .with_port(8000, 8000, "metrics");
 
-            local serviceRag =
-                engine.internalService(containerSetRag)
-                .with_port(8000, 8000, "metrics");
-
             engine.resources([
                 envSecrets,
                 containerSet,
-                containerSetRag,
                 service,
-                serviceRag,
             ])
 
     },

templates/components/cohere-rag.jsonnet

@@ -0,0 +1,56 @@
+local base = import "base/base.jsonnet";
+local images = import "values/images.jsonnet";
+local url = import "values/url.jsonnet";
+local prompts = import "prompts/mixtral.jsonnet";
+
+{
+
+    with:: function(key, value)
+        self + {
+            ["cohere-rag-" + key]:: value,
+        },
+
+    "cohere-rag-temperature":: 0.0,
+
+    "text-completion-rag" +: {
+    
+        create:: function(engine)
+
+            local envSecrets = engine.envSecrets("cohere-credentials")
+                .with_env_var("COHERE_KEY", "cohere-key");
+
+            local containerRag =
+                engine.container("text-completion-rag")
+                    .with_image(images.trustgraph_flow)
+                    .with_command([
+                        "text-completion-cohere",
+                        "-p",
+                        url.pulsar,
+                        "-t",
+                        "%0.3f" % $["cohere-rag-temperature"],
+                        "-i",
+                        "non-persistent://tg/request/text-completion-rag",
+                        "-o",
+                        "non-persistent://tg/response/text-completion-rag",
+                    ])
+                    .with_limits("0.5", "128M")
+                    .with_reservations("0.1", "128M");
+
+            local containerSetRag = engine.containers(
+                "text-completion-rag", [ containerRag ]
+            );
+
+            local serviceRag =
+                engine.internalService(containerSetRag)
+                .with_port(8000, 8000, "metrics");
+
+            engine.resources([
+                envSecrets,
+                containerSetRag,
+                serviceRag,
+            ])
+
+    },
+
+} + prompts
+

templates/components/cohere.jsonnet

@@ -5,9 +5,10 @@ local prompts = import "prompts/mixtral.jsonnet";
 
 {
 
-    // Override chunking
+    with:: function(key, value)
-    "chunk-size":: 150,
+        self + {
-    "chunk-overlap":: 10,
+            ["cohere-" + key]:: value,
+        },
 
     "cohere-temperature":: 0.0,
 
@@ -20,7 +21,7 @@ local prompts = import "prompts/mixtral.jsonnet";
 
             local container =
                 engine.container("text-completion")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "text-completion-cohere",
                         "-p",
@@ -31,45 +32,18 @@ local prompts = import "prompts/mixtral.jsonnet";
                     .with_limits("0.5", "128M")
                     .with_reservations("0.1", "128M");
 
-            local containerRag =
-                engine.container("text-completion-rag")
-                    .with_image(images.trustgraph)
-                    .with_command([
-                        "text-completion-cohere",
-                        "-p",
-                        url.pulsar,
-                        "-t",
-                        "%0.3f" % $["cohere-temperature"],
-                        "-i",
-                        "non-persistent://tg/request/text-completion-rag",
-                        "-o",
-                        "non-persistent://tg/response/text-completion-rag",
-                    ])
-                    .with_limits("0.5", "128M")
-                    .with_reservations("0.1", "128M");
-
             local containerSet = engine.containers(
                 "text-completion", [ container ]
             );
 
-            local containerSetRag = engine.containers(
-                "text-completion-rag", [ containerRag ]
-            );
-
             local service =
                 engine.internalService(containerSet)
                 .with_port(8000, 8000, "metrics");
 
-            local serviceRag =
-                engine.internalService(containerSetRag)
-                .with_port(8000, 8000, "metrics");
-
             engine.resources([
                 envSecrets,
                 containerSet,
-                containerSetRag,
                 service,
-                serviceRag,
             ])
 
     },

templates/components/document-rag.jsonnet

@@ -5,17 +5,21 @@ local prompts = import "prompts/mixtral.jsonnet";
 
 {
 
+    "document-rag-doc-limit":: 20,
+
     "document-rag" +: {
     
         create:: function(engine)
 
             local container =
                 engine.container("document-rag")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "document-rag",
                         "-p",
                         url.pulsar,
+                        "--doc-limit",
+                        std.toString($["document-rag-doc-limit"]),
                         "--prompt-request-queue",
                         "non-persistent://tg/request/prompt-rag",
                         "--prompt-response-queue",
@@ -39,5 +43,35 @@ local prompts = import "prompts/mixtral.jsonnet";
 
     },
 
+    "document-embeddings" +: {
+    
+        create:: function(engine)
+
+            local container =
+                engine.container("document-embeddings")
+                    .with_image(images.trustgraph_flow)
+                    .with_command([
+                        "document-embeddings",
+                        "-p",
+                        url.pulsar,
+                    ])
+                    .with_limits("1.0", "512M")
+                    .with_reservations("0.5", "512M");
+
+            local containerSet = engine.containers(
+                "document-embeddings", [ container ]
+            );
+
+            local service =
+                engine.internalService(containerSet)
+                .with_port(8000, 8000, "metrics");
+
+            engine.resources([
+                containerSet,
+                service,
+            ])
+
+    },
+
 }
 

templates/components/embeddings-fastembed.jsonnet

@@ -0,0 +1,43 @@
+local base = import "base/base.jsonnet";
+local images = import "values/images.jsonnet";
+local url = import "values/url.jsonnet";
+local prompts = import "prompts/mixtral.jsonnet";
+
+{
+
+    "embeddings-model":: "sentence-transformers/all-MiniLM-L6-v2",
+
+    embeddings +: {
+    
+        create:: function(engine)
+
+            local container =
+                engine.container("embeddings")
+                    .with_image(images.trustgraph_flow)
+                    .with_command([
+                        "embeddings-fastembed",
+                        "-p",
+                        url.pulsar,
+                        "-m",
+                        $["embeddings-model"],
+                    ])
+                    .with_limits("1.0", "400M")
+                    .with_reservations("0.5", "400M");
+
+            local containerSet = engine.containers(
+                "embeddings", [ container ]
+            );
+
+            local service =
+                engine.internalService(containerSet)
+                .with_port(8000, 8000, "metrics");
+
+            engine.resources([
+                containerSet,
+                service,
+            ])
+
+    },
+
+}
+

templates/components/embeddings-hf.jsonnet

@@ -13,7 +13,7 @@ local prompts = import "prompts/mixtral.jsonnet";
 
             local container =
                 engine.container("embeddings")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_hf)
                     .with_command([
                         "embeddings-hf",
                         "-p",

templates/components/embeddings-ollama.jsonnet

@@ -13,7 +13,7 @@ local url = import "values/url.jsonnet";
 
             local container =
                 engine.container("embeddings")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "embeddings-ollama",
                         "-p",

templates/components/falkordb.jsonnet

@@ -13,7 +13,7 @@ falkordb + {
 
             local container =
                 engine.container("store-triples")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "triples-write-falkordb",
                         "-p",
@@ -45,7 +45,7 @@ falkordb + {
 
             local container =
                 engine.container("query-triples")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "triples-query-falkordb",
                         "-p",

templates/components/googleaistudio-rag.jsonnet

@@ -0,0 +1,65 @@
+local base = import "base/base.jsonnet";
+local images = import "values/images.jsonnet";
+local url = import "values/url.jsonnet";
+local prompts = import "prompts/mixtral.jsonnet";
+
+{
+
+    with:: function(key, value)
+        self + {
+            ["googleaistudio-rag-" + key]:: value,
+        },
+
+    "googleaistudio-rag-max-output-tokens":: 4096,
+    "googleaistudio-rag-temperature":: 0.0,
+    "googleaistudio-rag-model":: "gemini-1.5-flash-002",
+
+    "text-completion-rag" +: {
+    
+        create:: function(engine)
+
+            local envSecrets = engine.envSecrets("googleaistudio-credentials")
+                .with_env_var("GOOGLE_AI_STUDIO_KEY", "googleaistudio-key");
+
+            local containerRag =
+                engine.container("text-completion-rag")
+                    .with_image(images.trustgraph_flow)
+                    .with_command([
+                        "text-completion-googleaistudio",
+                        "-p",
+                        url.pulsar,
+                        "-x",
+                        std.toString(
+                            $["googleaistudio-rag-max-output-tokens"]
+                        ),
+                        "-t",
+                        "%0.3f" % $["googleaistudio-rag-temperature"],
+                        "-m",
+                        $["googleaistudio-rag-model"],
+                        "-i",
+                        "non-persistent://tg/request/text-completion-rag",
+                        "-o",
+                        "non-persistent://tg/response/text-completion-rag",
+                    ])
+                    .with_env_var_secrets(envSecrets)
+                    .with_limits("0.5", "128M")
+                    .with_reservations("0.1", "128M");
+
+            local containerSetRag = engine.containers(
+                "text-completion-rag", [ containerRag ]
+            );
+
+            local serviceRag =
+                engine.internalService(containerSetRag)
+                .with_port(8000, 8000, "metrics");
+
+            engine.resources([
+                envSecrets,
+                containerSetRag,
+                serviceRag,
+            ])
+
+    },
+
+} + prompts
+

templates/components/googleaistudio.jsonnet

@@ -5,6 +5,11 @@ local prompts = import "prompts/mixtral.jsonnet";
 
 {
 
+    with:: function(key, value)
+        self + {
+            ["googleaistudio-" + key]:: value,
+        },
+
     "googleaistudio-max-output-tokens":: 4096,
     "googleaistudio-temperature":: 0.0,
     "googleaistudio-model":: "gemini-1.5-flash-002",
@@ -13,12 +18,12 @@ local prompts = import "prompts/mixtral.jsonnet";
     
         create:: function(engine)
 
-            local envSecrets = engine.envSecrets("googleaistudio-key")
+            local envSecrets = engine.envSecrets("googleaistudio-credentials")
                 .with_env_var("GOOGLE_AI_STUDIO_KEY", "googleaistudio-key");
 
             local container =
                 engine.container("text-completion")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "text-completion-googleaistudio",
                         "-p",
@@ -34,50 +39,18 @@ local prompts = import "prompts/mixtral.jsonnet";
                     .with_limits("0.5", "128M")
                     .with_reservations("0.1", "128M");
 
-            local containerRag =
-                engine.container("text-completion-rag")
-                    .with_image(images.trustgraph)
-                    .with_command([
-                        "text-completion-googleaistudio",
-                        "-p",
-                        url.pulsar,
-                        "-x",
-                        std.toString($["googleaistudio-max-output-tokens"]),
-                        "-t",
-                        "%0.3f" % $["googleaistudio-temperature"],
-                        "-m",
-                        $["googleaistudio-model"],
-                        "-i",
-                        "non-persistent://tg/request/text-completion-rag",
-                        "-o",
-                        "non-persistent://tg/response/text-completion-rag",
-                    ])
-                    .with_env_var_secrets(envSecrets)
-                    .with_limits("0.5", "128M")
-                    .with_reservations("0.1", "128M");
-
             local containerSet = engine.containers(
                 "text-completion", [ container ]
             );
 
-            local containerSetRag = engine.containers(
-                "text-completion-rag", [ containerRag ]
-            );
-
             local service =
                 engine.internalService(containerSet)
                 .with_port(8000, 8000, "metrics");
 
-            local serviceRag =
-                engine.internalService(containerSetRag)
-                .with_port(8000, 8000, "metrics");
-
             engine.resources([
                 envSecrets,
                 containerSet,
-                containerSetRag,
                 service,
-                serviceRag,
             ])
 
     },

templates/components/graph-rag.jsonnet

@@ -6,7 +6,8 @@ local url = import "values/url.jsonnet";
 
     "graph-rag-entity-limit":: 50,
     "graph-rag-triple-limit":: 30,
-    "graph-rag-max-subgraph-size":: 3000,
+    "graph-rag-max-subgraph-size":: 400,
+    "graph-rag-max-path-length":: 2,
 
     "kg-extract-definitions" +: {
     
@@ -14,7 +15,7 @@ local url = import "values/url.jsonnet";
 
             local container =
                 engine.container("kg-extract-definitions")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "kg-extract-definitions",
                         "-p",
@@ -44,7 +45,7 @@ local url = import "values/url.jsonnet";
 
             local container =
                 engine.container("kg-extract-relationships")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "kg-extract-relationships",
                         "-p",
@@ -74,7 +75,7 @@ local url = import "values/url.jsonnet";
 
             local container =
                 engine.container("kg-extract-topics")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "kg-extract-topics",
                         "-p",
@@ -104,7 +105,7 @@ local url = import "values/url.jsonnet";
 
             local container =
                 engine.container("graph-rag")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "graph-rag",
                         "-p",
@@ -119,6 +120,8 @@ local url = import "values/url.jsonnet";
                         std.toString($["graph-rag-triple-limit"]),
                         "--max-subgraph-size",
                         std.toString($["graph-rag-max-subgraph-size"]),
+                        "--max-path-length",
+                        std.toString($["graph-rag-max-path-length"]),
                     ])
                     .with_limits("0.5", "128M")
                     .with_reservations("0.1", "128M");
@@ -138,5 +141,35 @@ local url = import "values/url.jsonnet";
 
     },
 
+    "graph-embeddings" +: {
+    
+        create:: function(engine)
+
+            local container =
+                engine.container("graph-embeddings")
+                    .with_image(images.trustgraph_flow)
+                    .with_command([
+                        "graph-embeddings",
+                        "-p",
+                        url.pulsar,
+                    ])
+                    .with_limits("1.0", "512M")
+                    .with_reservations("0.5", "512M");
+
+            local containerSet = engine.containers(
+                "graph-embeddings", [ container ]
+            );
+
+            local service =
+                engine.internalService(containerSet)
+                .with_port(8000, 8000, "metrics");
+
+            engine.resources([
+                containerSet,
+                service,
+            ])
+
+    },
+
 }
 

templates/components/librarian.jsonnet

@@ -0,0 +1,43 @@
+local base = import "base/base.jsonnet";
+local images = import "values/images.jsonnet";
+local url = import "values/url.jsonnet";
+local minio = import "stores/minio.jsonnet";
+local cassandra = import "stores/cassandra.jsonnet";
+
+{
+
+    "librarian" +: {
+    
+        create:: function(engine)
+
+            local container =
+                engine.container("librarian")
+                    .with_image(images.trustgraph_flow)
+                    .with_command([
+                        "librarian",
+                        "-p",
+                        url.pulsar,
+                    ])
+                    .with_limits("0.5", "256M")
+                    .with_reservations("0.1", "256M");
+
+            local containerSet = engine.containers(
+                "librarian", [ container ]
+            );
+
+            local service =
+                engine.internalService(containerSet)
+                .with_port(8000, 8000, "metrics");
+
+            engine.resources([
+                containerSet,
+                service,
+            ])
+
+    },
+
+}
+
+    // Minio and Cassandra are used by the Librarian
+    + minio + cassandra
+

templates/components/llamafile-rag.jsonnet

@@ -0,0 +1,57 @@
+local base = import "base/base.jsonnet";
+local images = import "values/images.jsonnet";
+local url = import "values/url.jsonnet";
+local prompts = import "prompts/slm.jsonnet";
+
+{
+
+    with:: function(key, value)
+        self + {
+            ["llamafile-rag-" + key]:: value,
+        },
+
+    "llamafile-rag-model":: "LLaMA_CPP",
+
+    "text-completion-rag" +: {
+    
+        create:: function(engine)
+
+            local envSecrets = engine.envSecrets("llamafile-credentials")
+                .with_env_var("LLAMAFILE_URL", "llamafile-url");
+
+            local containerRag =
+                engine.container("text-completion-rag")
+                    .with_image(images.trustgraph_flow)
+                    .with_command([
+                        "text-completion-llamafile",
+                        "-p",
+                        url.pulsar,
+                        "-m",
+                        $["llamafile-rag-model"],
+                        "-i",
+                        "non-persistent://tg/request/text-completion-rag",
+                        "-o",
+                        "non-persistent://tg/response/text-completion-rag",
+                    ])
+                    .with_env_var_secrets(envSecrets)
+                    .with_limits("0.5", "128M")
+                    .with_reservations("0.1", "128M");
+
+            local containerSetRag = engine.containers(
+                "text-completion-rag", [ containerRag ]
+            );
+
+            local serviceRag =
+                engine.internalService(containerSetRag)
+                .with_port(8080, 8080, "metrics");
+
+            engine.resources([
+                envSecrets,
+                containerSetRag,
+                serviceRag,
+            ])
+
+    },
+
+} + prompts
+

templates/components/llamafile.jsonnet

@@ -5,6 +5,11 @@ local prompts = import "prompts/slm.jsonnet";
 
 {
 
+    with:: function(key, value)
+        self + {
+            ["llamafile-" + key]:: value,
+        },
+
     "llamafile-model":: "LLaMA_CPP",
 
     "text-completion" +: {
@@ -16,7 +21,7 @@ local prompts = import "prompts/slm.jsonnet";
 
             local container =
                 engine.container("text-completion")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "text-completion-llamafile",
                         "-p",
@@ -28,46 +33,18 @@ local prompts = import "prompts/slm.jsonnet";
                     .with_limits("0.5", "128M")
                     .with_reservations("0.1", "128M");
 
-            local containerRag =
-                engine.container("text-completion-rag")
-                    .with_image(images.trustgraph)
-                    .with_command([
-                        "text-completion-llamafile",
-                        "-p",
-                        url.pulsar,
-                        "-m",
-                        $["llamafile-model"],
-                        "-i",
-                        "non-persistent://tg/request/text-completion-rag",
-                        "-o",
-                        "non-persistent://tg/response/text-completion-rag",
-                    ])
-                    .with_env_var_secrets(envSecrets)
-                    .with_limits("0.5", "128M")
-                    .with_reservations("0.1", "128M");
-
             local containerSet = engine.containers(
                 "text-completion", [ container ]
             );
 
-            local containerSetRag = engine.containers(
-                "text-completion-rag", [ containerRag ]
-            );
-
             local service =
                 engine.internalService(containerSet)
                 .with_port(8080, 8080, "metrics");
 
-            local serviceRag =
-                engine.internalService(containerSetRag)
-                .with_port(8080, 8080, "metrics");
-
             engine.resources([
                 envSecrets,
                 containerSet,
-                containerSetRag,
                 service,
-                serviceRag,
             ])
 
     },

templates/components/memgraph.jsonnet

@@ -14,7 +14,7 @@ memgraph + {
 
             local container =
                 engine.container("store-triples")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "triples-write-memgraph",
                         "-p",
@@ -48,7 +48,7 @@ memgraph + {
 
             local container =
                 engine.container("query-triples")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "triples-query-memgraph",
                         "-p",

templates/components/milvus.jsonnet

@@ -12,7 +12,7 @@ milvus + {
 
             local container =
                 engine.container("store-graph-embeddings")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "ge-write-milvus",
                         "-p",
@@ -44,7 +44,7 @@ milvus + {
 
             local container =
                 engine.container("query-graph-embeddings")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "ge-query-milvus",
                         "-p",
@@ -76,7 +76,7 @@ milvus + {
 
             local container =
                 engine.container("store-doc-embeddings")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "de-write-milvus",
                         "-p",
@@ -108,7 +108,7 @@ milvus + {
 
             local container =
                 engine.container("query-doc-embeddings")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "de-query-milvus",
                         "-p",

templates/components/mistral-ocr.jsonnet

@@ -0,0 +1,47 @@
+local images = import "values/images.jsonnet";
+local url = import "values/url.jsonnet";
+
+{
+
+    with:: function(key, value)
+        self + {
+            ["mistral-" + key]:: value,
+        },
+
+    "pdf-decoder" +: {
+    
+        create:: function(engine)
+
+            local envSecrets = engine.envSecrets("mistral-credentials")
+                .with_env_var("MISTRAL_TOKEN", "mistral-token");
+
+            local container =
+                engine.container("mistral-ocr")
+                    .with_image(images.trustgraph_flow)
+                    .with_command([
+                        "pdf-ocr-mistral",
+                        "-p",
+                        url.pulsar,
+                    ])
+                    .with_env_var_secrets(envSecrets)
+                    .with_limits("0.5", "128M")
+                    .with_reservations("0.1", "128M");
+
+            local containerSet = engine.containers(
+                "mistral-ocr", [ container ]
+            );
+
+            local service =
+                engine.internalService(containerSet)
+                .with_port(8080, 8080, "metrics");
+
+            engine.resources([
+                envSecrets,
+                containerSet,
+                service,
+            ])
+
+    },
+
+} + prompts
+

templates/components/mistral-rag.jsonnet

@@ -0,0 +1,63 @@
+local base = import "base/base.jsonnet";
+local images = import "values/images.jsonnet";
+local url = import "values/url.jsonnet";
+local prompts = import "prompts/mixtral.jsonnet";
+
+{
+
+    with:: function(key, value)
+        self + {
+            ["mistral-rag-" + key]:: value,
+        },
+
+    "mistral-rag-max-output-tokens":: 4096,
+    "mistral-rag-temperature":: 0.0,
+    "mistral-rag-model":: "ministral-8b-latest",
+
+    "text-completion-rag" +: {
+    
+        create:: function(engine)
+
+            local envSecrets = engine.envSecrets("mistral-credentials")
+                .with_env_var("MISTRAL_TOKEN", "mistral-token");
+
+            local containerRag =
+                engine.container("text-completion-rag")
+                    .with_image(images.trustgraph_flow)
+                    .with_command([
+                        "text-completion-mistral",
+                        "-p",
+                        url.pulsar,
+                        "-x",
+                        std.toString($["mistral-rag-max-output-tokens"]),
+                        "-t",
+                        "%0.3f" % $["mistral-rag-temperature"],
+                        "-m",
+                        $["mistral-rag-model"],
+                        "-i",
+                        "non-persistent://tg/request/text-completion-rag",
+                        "-o",
+                        "non-persistent://tg/response/text-completion-rag",
+                    ])
+                    .with_env_var_secrets(envSecrets)
+                    .with_limits("0.5", "128M")
+                    .with_reservations("0.1", "128M");
+
+            local containerSetRag = engine.containers(
+                "text-completion-rag", [ containerRag ]
+            );
+
+            local serviceRag =
+                engine.internalService(containerSetRag)
+                .with_port(8080, 8080, "metrics");
+
+            engine.resources([
+                envSecrets,
+                containerSetRag,
+                serviceRag,
+            ])
+
+    },
+
+} + prompts
+

templates/components/mistral.jsonnet

@@ -0,0 +1,59 @@
+local base = import "base/base.jsonnet";
+local images = import "values/images.jsonnet";
+local url = import "values/url.jsonnet";
+local prompts = import "prompts/mixtral.jsonnet";
+
+{
+
+    with:: function(key, value)
+        self + {
+            ["mistral-" + key]:: value,
+        },
+
+    "mistral-max-output-tokens":: 4096,
+    "mistral-temperature":: 0.0,
+    "mistral-model":: "ministral-8b-latest",
+
+    "text-completion" +: {
+    
+        create:: function(engine)
+
+            local envSecrets = engine.envSecrets("mistral-credentials")
+                .with_env_var("MISTRAL_TOKEN", "mistral-token");
+
+            local container =
+                engine.container("text-completion")
+                    .with_image(images.trustgraph_flow)
+                    .with_command([
+                        "text-completion-mistral",
+                        "-p",
+                        url.pulsar,
+                        "-x",
+                        std.toString($["mistral-max-output-tokens"]),
+                        "-t",
+                        "%0.3f" % $["mistral-temperature"],
+                        "-m",
+                        $["mistral-model"],
+                    ])
+                    .with_env_var_secrets(envSecrets)
+                    .with_limits("0.5", "128M")
+                    .with_reservations("0.1", "128M");
+
+            local containerSet = engine.containers(
+                "text-completion", [ container ]
+            );
+
+            local service =
+                engine.internalService(containerSet)
+                .with_port(8080, 8080, "metrics");
+
+            engine.resources([
+                envSecrets,
+                containerSet,
+                service,
+            ])
+
+    },
+
+} + prompts
+

templates/components/neo4j.jsonnet

@@ -13,7 +13,7 @@ neo4j + {
 
             local container =
                 engine.container("store-triples")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "triples-write-neo4j",
                         "-p",
@@ -45,7 +45,7 @@ neo4j + {
 
             local container =
                 engine.container("query-triples")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "triples-query-neo4j",
                         "-p",

templates/components/ocr.jsonnet

@@ -0,0 +1,38 @@
+local images = import "values/images.jsonnet";
+local url = import "values/url.jsonnet";
+
+{
+
+    "pdf-decoder" +: {
+    
+        create:: function(engine)
+
+            local container =
+                engine.container("pdf-ocr")
+                    .with_image(images.trustgraph_ocr)
+                    .with_command([
+                        "pdf-ocr",
+                        "-p",
+                        url.pulsar,
+                    ])
+                    .with_limits("1.0", "512M")
+                    .with_reservations("0.1", "512M");
+
+            local containerSet = engine.containers(
+                "pdf-ocr", [ container ]
+            );
+
+            local service =
+                engine.internalService(containerSet)
+                .with_port(8080, 8080, "metrics");
+
+            engine.resources([
+                envSecrets,
+                containerSet,
+                service,
+            ])
+
+    },
+
+} + prompts
+

templates/components/ollama-rag.jsonnet

@@ -0,0 +1,57 @@
+local base = import "base/base.jsonnet";
+local images = import "values/images.jsonnet";
+local url = import "values/url.jsonnet";
+local prompts = import "prompts/mixtral.jsonnet";
+
+{
+
+    with:: function(key, value)
+        self + {
+            ["ollama-rag-" + key]:: value,
+        },
+
+    "ollama-rag-model":: "gemma2:9b",
+
+    "text-completion-rag" +: {
+    
+        create:: function(engine)
+
+            local envSecrets = engine.envSecrets("ollama-credentials")
+                .with_env_var("OLLAMA_HOST", "ollama-host");
+
+            local containerRag =
+                engine.container("text-completion-rag")
+                    .with_image(images.trustgraph_flow)
+                    .with_command([
+                        "text-completion-ollama",
+                        "-p",
+                        url.pulsar,
+                        "-m",
+                        $["ollama-rag-model"],
+                        "-i",
+                        "non-persistent://tg/request/text-completion-rag",
+                        "-o",
+                        "non-persistent://tg/response/text-completion-rag",
+                    ])
+                    .with_env_var_secrets(envSecrets)
+                    .with_limits("0.5", "128M")
+                    .with_reservations("0.1", "128M");
+
+            local containerSetRag = engine.containers(
+                "text-completion-rag", [ containerRag ]
+            );
+
+            local serviceRag =
+                engine.internalService(containerSetRag)
+                .with_port(8080, 8080, "metrics");
+
+            engine.resources([
+                envSecrets,
+                containerSetRag,
+                serviceRag,
+            ])
+
+    },
+
+} + prompts
+

templates/components/ollama.jsonnet

@@ -5,6 +5,11 @@ local prompts = import "prompts/mixtral.jsonnet";
 
 {
 
+    with:: function(key, value)
+        self + {
+            ["ollama-" + key]:: value,
+        },
+
     "ollama-model":: "gemma2:9b",
 
     "text-completion" +: {
@@ -16,7 +21,7 @@ local prompts = import "prompts/mixtral.jsonnet";
 
             local container =
                 engine.container("text-completion")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "text-completion-ollama",
                         "-p",
@@ -28,46 +33,18 @@ local prompts = import "prompts/mixtral.jsonnet";
                     .with_limits("0.5", "128M")
                     .with_reservations("0.1", "128M");
 
-            local containerRag =
-                engine.container("text-completion-rag")
-                    .with_image(images.trustgraph)
-                    .with_command([
-                        "text-completion-ollama",
-                        "-p",
-                        url.pulsar,
-                        "-m",
-                        $["ollama-model"],
-                        "-i",
-                        "non-persistent://tg/request/text-completion-rag",
-                        "-o",
-                        "non-persistent://tg/response/text-completion-rag",
-                    ])
-                    .with_env_var_secrets(envSecrets)
-                    .with_limits("0.5", "128M")
-                    .with_reservations("0.1", "128M");
-
             local containerSet = engine.containers(
                 "text-completion", [ container ]
             );
 
-            local containerSetRag = engine.containers(
-                "text-completion-rag", [ containerRag ]
-            );
-
             local service =
                 engine.internalService(containerSet)
                 .with_port(8080, 8080, "metrics");
 
-            local serviceRag =
-                engine.internalService(containerSetRag)
-                .with_port(8080, 8080, "metrics");
-
             engine.resources([
                 envSecrets,
                 containerSet,
-                containerSetRag,
                 service,
-                serviceRag,
             ])
 
     },

templates/components/openai-rag.jsonnet

@@ -0,0 +1,63 @@
+local base = import "base/base.jsonnet";
+local images = import "values/images.jsonnet";
+local url = import "values/url.jsonnet";
+local prompts = import "prompts/mixtral.jsonnet";
+
+{
+
+    with:: function(key, value)
+        self + {
+            ["openai-rag-" + key]:: value,
+        },
+
+    "openai-rag-max-output-tokens":: 4096,
+    "openai-rag-temperature":: 0.0,
+    "openai-rag-model":: "GPT-3.5-Turbo",
+
+    "text-completion-rag" +: {
+    
+        create:: function(engine)
+
+            local envSecrets = engine.envSecrets("openai-credentials")
+                .with_env_var("OPENAI_TOKEN", "openai-token");
+
+            local containerRag =
+                engine.container("text-completion-rag")
+                    .with_image(images.trustgraph_flow)
+                    .with_command([
+                        "text-completion-openai",
+                        "-p",
+                        url.pulsar,
+                        "-x",
+                        std.toString($["openai-rag-max-output-tokens"]),
+                        "-t",
+                        "%0.3f" % $["openai-rag-temperature"],
+                        "-m",
+                        $["openai-rag-model"],
+                        "-i",
+                        "non-persistent://tg/request/text-completion-rag",
+                        "-o",
+                        "non-persistent://tg/response/text-completion-rag",
+                    ])
+                    .with_env_var_secrets(envSecrets)
+                    .with_limits("0.5", "128M")
+                    .with_reservations("0.1", "128M");
+
+            local containerSetRag = engine.containers(
+                "text-completion-rag", [ containerRag ]
+            );
+
+            local serviceRag =
+                engine.internalService(containerSetRag)
+                .with_port(8080, 8080, "metrics");
+
+            engine.resources([
+                envSecrets,
+                containerSetRag,
+                serviceRag,
+            ])
+
+    },
+
+} + prompts
+

templates/components/openai.jsonnet

@@ -5,6 +5,11 @@ local prompts = import "prompts/mixtral.jsonnet";
 
 {
 
+    with:: function(key, value)
+        self + {
+            ["openai-" + key]:: value,
+        },
+
     "openai-max-output-tokens":: 4096,
     "openai-temperature":: 0.0,
     "openai-model":: "GPT-3.5-Turbo",
@@ -18,7 +23,7 @@ local prompts = import "prompts/mixtral.jsonnet";
 
             local container =
                 engine.container("text-completion")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "text-completion-openai",
                         "-p",
@@ -34,50 +39,18 @@ local prompts = import "prompts/mixtral.jsonnet";
                     .with_limits("0.5", "128M")
                     .with_reservations("0.1", "128M");
 
-            local containerRag =
-                engine.container("text-completion-rag")
-                    .with_image(images.trustgraph)
-                    .with_command([
-                        "text-completion-openai",
-                        "-p",
-                        url.pulsar,
-                        "-x",
-                        std.toString($["openai-max-output-tokens"]),
-                        "-t",
-                        "%0.3f" % $["openai-temperature"],
-                        "-m",
-                        $["openai-model"],
-                        "-i",
-                        "non-persistent://tg/request/text-completion-rag",
-                        "-o",
-                        "non-persistent://tg/response/text-completion-rag",
-                    ])
-                    .with_env_var_secrets(envSecrets)
-                    .with_limits("0.5", "128M")
-                    .with_reservations("0.1", "128M");
-
             local containerSet = engine.containers(
                 "text-completion", [ container ]
             );
 
-            local containerSetRag = engine.containers(
-                "text-completion-rag", [ containerRag ]
-            );
-
             local service =
                 engine.internalService(containerSet)
                 .with_port(8080, 8080, "metrics");
 
-            local serviceRag =
-                engine.internalService(containerSetRag)
-                .with_port(8080, 8080, "metrics");
-
             engine.resources([
                 envSecrets,
                 containerSet,
-                containerSetRag,
                 service,
-                serviceRag,
             ])
 
     },

templates/components/pinecone.jsonnet

@@ -17,7 +17,7 @@ local cassandra_hosts = "cassandra";
 
             local container =
                 engine.container("store-graph-embeddings")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "ge-write-pinecone",
                         "-p",
@@ -52,7 +52,7 @@ local cassandra_hosts = "cassandra";
 
             local container =
                 engine.container("query-graph-embeddings")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "ge-query-pinecone",
                         "-p",
@@ -87,7 +87,7 @@ local cassandra_hosts = "cassandra";
 
             local container =
                 engine.container("store-doc-embeddings")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "de-write-pinecone",
                         "-p",
@@ -122,7 +122,7 @@ local cassandra_hosts = "cassandra";
 
             local container =
                 engine.container("query-doc-embeddings")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "de-query-pinecone",
                         "-p",

templates/components/prompt-overrides.jsonnet

@@ -1,7 +1,3 @@
-local base = import "base/base.jsonnet";
-local images = import "values/images.jsonnet";
-local url = import "values/url.jsonnet";
-local prompts = import "prompts/mixtral.jsonnet";
 local default_prompts = import "prompts/default-prompts.jsonnet";
 
 {

templates/components/prompt-template.jsonnet

@@ -44,7 +44,7 @@ local default_prompts = import "prompts/default-prompts.jsonnet";
 
             local container =
                 engine.container("prompt")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "prompt-template",
                         "-p",
@@ -84,7 +84,7 @@ local default_prompts = import "prompts/default-prompts.jsonnet";
 
             local container =
                 engine.container("prompt-rag")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "prompt-template",
                         "-p",

templates/components/pulsar.jsonnet

@@ -2,32 +2,114 @@ local base = import "base/base.jsonnet";
 local images = import "values/images.jsonnet";
 local url = import "values/url.jsonnet";
 
+// This is a Pulsar configuration.  Non-standalone mode so we deploy
+// individual components: bookkeeper, broker and zookeeper.
+// 
+// This also deploys the TrustGraph 'admin' container which initialises
+// TrustGraph-specific namespaces etc.
+
 {
 
     "pulsar" +: {
 
         create:: function(engine)
 
-//            local confVolume = engine.volume("pulsar-conf").with_size("2G");
+            // Zookeeper volume
-            local dataVolume = engine.volume("pulsar-data").with_size("20G");
+            local zkVolume = engine.volume("zookeeper").with_size("1G");
 
-            local container =
+            // Zookeeper container
+            local zkContainer = 
+                engine.container("zookeeper")
+                    .with_image(images.pulsar)
+                    .with_command([
+                        "bash",
+                        "-c",
+                        "bin/apply-config-from-env.py conf/zookeeper.conf && bin/generate-zookeeper-config.sh conf/zookeeper.conf && exec bin/pulsar zookeeper"
+                    ])
+                    .with_limits("1", "400M")
+                    .with_reservations("0.05", "400M")
+                    .with_user("0:1000")
+                    .with_volume_mount(zkVolume, "/pulsar/data/zookeeper")
+                    .with_environment({
+                        "metadataStoreUrl": "zk:zookeeper:2181",
+                        "PULSAR_MEM": "-Xms256m -Xmx256m -XX:MaxDirectMemorySize=256m",
+                    })
+                    .with_port(2181, 2181, "zookeeper")
+                    .with_port(2888, 2888, "zookeeper2")
+                    .with_port(3888, 3888, "zookeeper3");
+
+            // Pulsar cluster init container
+            local initContainer =
+                engine.container("pulsar-init")
+                    .with_image(images.pulsar)
+                    .with_command([
+                        "bash",
+                        "-c",
+                        "sleep 10 && bin/pulsar initialize-cluster-metadata --cluster cluster-a --zookeeper zookeeper:2181 --configuration-store zookeeper:2181 --web-service-url http://pulsar:8080 --broker-service-url pulsar://pulsar:6650",
+                    ])
+                    .with_limits("1", "512M")
+                    .with_reservations("0.05", "512M")
+                    .with_environment({
+                        "PULSAR_MEM": "-Xms256m -Xmx256m -XX:MaxDirectMemorySize=256m",
+                    });
+
+
+            // Bookkeeper volume
+            local bookieVolume = engine.volume("bookie").with_size("20G");
+
+            // Bookkeeper container
+            local bookieContainer = 
+                engine.container("bookie")
+                    .with_image(images.pulsar)
+                    .with_command([
+                        "bash",
+                        "-c",
+                        "bin/apply-config-from-env.py conf/bookkeeper.conf && exec bin/pulsar bookie"
+                        // false ^ causes this to be a 'failure' exit.
+                    ])
+                    .with_limits("1", "800M")
+                    .with_reservations("0.1", "800M")
+                    .with_user("0:1000")
+                    .with_volume_mount(bookieVolume, "/pulsar/data/bookkeeper")
+                    .with_environment({
+                        "clusterName": "cluster-a",
+                        "zkServers": "zookeeper:2181",
+                        "bookieId": "bookie",
+                        "metadataStoreUri": "metadata-store:zk:zookeeper:2181",
+                        "advertisedAddress": "bookie",
+                        "BOOKIE_MEM": "-Xms512m -Xmx512m -XX:MaxDirectMemorySize=256m",
+                    })
+                    .with_port(3181, 3181, "bookie");
+
+            // Pulsar broker, stateless (uses ZK and Bookkeeper for state)
+            local brokerContainer = 
                 engine.container("pulsar")
                     .with_image(images.pulsar)
-                    .with_command(["bin/pulsar", "standalone"])
+                    .with_command([
+                        "bash",
+                        "-c",
+                        "bin/apply-config-from-env.py conf/broker.conf && exec bin/pulsar broker"
+                    ])
+                    .with_limits("1", "800M")
+                    .with_reservations("0.1", "800M")
                     .with_environment({
-                        "PULSAR_MEM": "-Xms600M -Xmx600M"
+                        "metadataStoreUrl": "zk:zookeeper:2181",
+                        "zookeeperServers": "zookeeper:2181",
+                        "clusterName": "cluster-a",
+                        "managedLedgerDefaultEnsembleSize": "1",
+                        "managedLedgerDefaultWriteQuorum": "1",
+                        "managedLedgerDefaultAckQuorum": "1",
+                        "advertisedAddress": "pulsar",
+                        "advertisedListeners": "external:pulsar://pulsar:6650,localhost:pulsar://localhost:6650",
+                        "PULSAR_MEM": "-Xms512m -Xmx512m -XX:MaxDirectMemorySize=256m",
                     })
-                    .with_limits("2.0", "1500M")
+                    .with_port(6650, 6650, "pulsar")
-                    .with_reservations("1.0", "1500M")
+                    .with_port(8080, 8080, "admin");
-//                    .with_volume_mount(confVolume, "/pulsar/conf")
-                    .with_volume_mount(dataVolume, "/pulsar/data")
-                    .with_port(6650, 6650, "bookie")
-                    .with_port(8080, 8080, "http");
 
+            // Trustgraph Pulsar initialisation
             local adminContainer =
-                engine.container("init-pulsar")
+                engine.container("init-trustgraph")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "tg-init-pulsar",
                         "-p",
@@ -36,10 +118,32 @@ local url = import "values/url.jsonnet";
                     .with_limits("1", "128M")
                     .with_reservations("0.1", "128M");
 
-            local containerSet = engine.containers(
+            // Container sets
+            local zkContainerSet = engine.containers(
+                "zookeeper",
+                [
+                    zkContainer,
+                ]
+            );
+
+            local initContainerSet = engine.containers(
+                "init-pulsar",
+                [
+                    initContainer,
+                ]
+            );
+
+            local bookieContainerSet = engine.containers(
+                "bookie",
+                [
+                    bookieContainer,
+                ]
+            );
+
+            local brokerContainerSet = engine.containers(
                 "pulsar",
                 [
-                    container
+                    brokerContainer,
                 ]
             );
 
@@ -50,17 +154,35 @@ local url = import "values/url.jsonnet";
                 ]
             );
 
-            local service =
+            // Zookeeper service
-                engine.service(containerSet)
+            local zkService =
-                .with_port(6650, 6650, "bookie")
+                engine.service(zkContainerSet)
-                .with_port(8080, 8080, "http");
+                .with_port(2181, 2181, "zookeeper")
+                .with_port(2888, 2888, "zookeeper2")
+                .with_port(3888, 3888, "zookeeper3");
+
+            // Bookkeeper service
+            local bookieService =
+                engine.service(bookieContainerSet)
+                .with_port(3181, 3181, "bookie");
+
+            // Pulsar broker service
+            local brokerService =
+                engine.service(brokerContainerSet)
+                .with_port(6650, 6650, "pulsar")
+                .with_port(8080, 8080, "admin");
 
             engine.resources([
-//                confVolume,
+                zkVolume,
-                dataVolume,
+                bookieVolume,
-                containerSet,
+                zkContainerSet,
+                initContainerSet,
+                bookieContainerSet,
+                brokerContainerSet,
                 adminContainerSet,
-                service,
+                zkService,
+                bookieService,
+                brokerService,
             ])
 
     }

templates/components/qdrant.jsonnet

@@ -12,7 +12,7 @@ qdrant + {
 
             local container =
                 engine.container("store-graph-embeddings")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "ge-write-qdrant",
                         "-p",
@@ -44,7 +44,7 @@ qdrant + {
 
             local container =
                 engine.container("query-graph-embeddings")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "ge-query-qdrant",
                         "-p",
@@ -76,7 +76,7 @@ qdrant + {
 
             local container =
                 engine.container("store-doc-embeddings")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "de-write-qdrant",
                         "-p",
@@ -108,7 +108,7 @@ qdrant + {
 
             local container =
                 engine.container("query-doc-embeddings")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "de-query-qdrant",
                         "-p",

templates/components/trustgraph.jsonnet

@@ -1,7 +1,6 @@
 local base = import "base/base.jsonnet";
 local images = import "values/images.jsonnet";
 local url = import "values/url.jsonnet";
-local prompt = import "prompt-template.jsonnet";
 
 {
 
@@ -22,7 +21,7 @@ local prompt = import "prompt-template.jsonnet";
 
             local container =
                 engine.container("api-gateway")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "api-gateway",
                         "-p",
@@ -61,7 +60,7 @@ local prompt = import "prompt-template.jsonnet";
 
             local container =
                 engine.container("chunker")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "chunker-token",
                         "-p",
@@ -95,7 +94,7 @@ local prompt = import "prompt-template.jsonnet";
 
             local container =
                 engine.container("pdf-decoder")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "pdf-decoder",
                         "-p",
@@ -119,43 +118,13 @@ local prompt = import "prompt-template.jsonnet";
 
     },
 
-    "vectorize" +: {
-    
-        create:: function(engine)
-
-            local container =
-                engine.container("vectorize")
-                    .with_image(images.trustgraph)
-                    .with_command([
-                        "embeddings-vectorize",
-                        "-p",
-                        url.pulsar,
-                    ])
-                    .with_limits("1.0", "512M")
-                    .with_reservations("0.5", "512M");
-
-            local containerSet = engine.containers(
-                "vectorize", [ container ]
-            );
-
-            local service =
-                engine.internalService(containerSet)
-                .with_port(8000, 8000, "metrics");
-
-            engine.resources([
-                containerSet,
-                service,
-            ])
-
-    },
-
     "metering" +: {
     
         create:: function(engine)
 
             local container =
                 engine.container("metering")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "metering",
                         "-p",
@@ -185,7 +154,7 @@ local prompt = import "prompt-template.jsonnet";
 
             local container =
                 engine.container("metering-rag")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_flow)
                     .with_command([
                         "metering",
                         "-p",
@@ -211,5 +180,5 @@ local prompt = import "prompt-template.jsonnet";
 
     },
 
-} + prompt
+}
 

templates/components/vertexai-rag.jsonnet

@@ -0,0 +1,74 @@
+local base = import "base/base.jsonnet";
+local images = import "values/images.jsonnet";
+local url = import "values/url.jsonnet";
+local prompts = import "prompts/mixtral.jsonnet";
+
+{
+
+    with:: function(key, value)
+        self + {
+            ["vertexai-rag-" + key]:: value,
+        },
+
+    "vertexai-rag-model":: "gemini-1.0-pro-001",
+    "vertexai-rag-private-key":: "/vertexai/private.json",
+    "vertexai-rag-region":: "us-central1",
+    "vertexai-rag-max-output-tokens":: 4096,
+    "vertexai-rag-temperature":: 0.0,
+
+    "text-completion-rag" +: {
+    
+        create:: function(engine)
+
+            local cfgVol = engine.secretVolume(
+	        "vertexai-creds",
+	        "./vertexai",
+		{
+		    "private.json": importstr "vertexai/private.json",
+		}
+            );
+
+            local container =
+                engine.container("text-completion-rag")
+                    .with_image(images.trustgraph_vertexai)
+                    .with_command([
+                        "text-completion-vertexai",
+                        "-p",
+                        url.pulsar,
+                        "-k",
+                        $["vertexai-rag-private-key"],
+                        "-r",
+                        $["vertexai-rag-region"],
+                        "-x",
+                        std.toString($["vertexai-rag-max-output-tokens"]),
+                        "-t",
+                        "%0.3f" % $["vertexai-rag-temperature"],
+                        "-m",
+                        $["vertexai-rag-model"],
+                        "-i",
+                        "non-persistent://tg/request/text-completion-rag",
+                        "-o",
+                        "non-persistent://tg/response/text-completion-rag",
+                    ])
+                    .with_limits("0.5", "256M")
+                    .with_reservations("0.1", "256M")
+                    .with_volume_mount(cfgVol, "/vertexai");
+
+            local containerSet = engine.containers(
+                "text-completion-rag", [ container ]
+            );
+
+            local service =
+                engine.internalService(containerSet)
+                .with_port(8000, 8000, "metrics");
+
+            engine.resources([
+                cfgVol,
+                containerSet,
+                service,
+            ])
+
+    }
+
+} + prompts
+

templates/components/vertexai.jsonnet

@@ -5,6 +5,11 @@ local prompts = import "prompts/mixtral.jsonnet";
 
 {
 
+    with:: function(key, value)
+        self + {
+            ["vertexai-" + key]:: value,
+        },
+
     "vertexai-model":: "gemini-1.0-pro-001",
     "vertexai-private-key":: "/vertexai/private.json",
     "vertexai-region":: "us-central1",
@@ -25,7 +30,7 @@ local prompts = import "prompts/mixtral.jsonnet";
 
             local container =
                 engine.container("text-completion")
-                    .with_image(images.trustgraph)
+                    .with_image(images.trustgraph_vertexai)
                     .with_command([
                         "text-completion-vertexai",
                         "-p",
@@ -61,59 +66,5 @@ local prompts = import "prompts/mixtral.jsonnet";
 
     },
 
-    "text-completion-rag" +: {
-    
-        create:: function(engine)
-
-            local cfgVol = engine.secretVolume(
-	        "vertexai-creds",
-	        "./vertexai",
-		{
-		    "private.json": importstr "vertexai/private.json",
-		}
-            );
-
-            local container =
-                engine.container("text-completion-rag")
-                    .with_image(images.trustgraph)
-                    .with_command([
-                        "text-completion-vertexai",
-                        "-p",
-                        url.pulsar,
-                        "-k",
-                        $["vertexai-private-key"],
-                        "-r",
-                        $["vertexai-region"],
-                        "-x",
-                        std.toString($["vertexai-max-output-tokens"]),
-                        "-t",
-                        "%0.3f" % $["vertexai-temperature"],
-                        "-m",
-                        $["vertexai-model"],
-                        "-i",
-                        "non-persistent://tg/request/text-completion-rag",
-                        "-o",
-                        "non-persistent://tg/response/text-completion-rag",
-                    ])
-                    .with_limits("0.5", "256M")
-                    .with_reservations("0.1", "256M")
-                    .with_volume_mount(cfgVol, "/vertexai");
-
-            local containerSet = engine.containers(
-                "text-completion-rag", [ container ]
-            );
-
-            local service =
-                engine.internalService(containerSet)
-                .with_port(8000, 8000, "metrics");
-
-            engine.resources([
-                cfgVol,
-                containerSet,
-                service,
-            ])
-
-    }
-
 } + prompts
 

templates/config-to-aks-k8s.jsonnet

@@ -0,0 +1,16 @@
+
+local engine = import "engine/aks-k8s.jsonnet";
+local decode = import "util/decode-config.jsonnet";
+local components = import "components.jsonnet";
+
+// Import config
+local config = import "config.json";
+
+// Produce patterns from config
+local patterns = decode(config);
+
+// Extract resources usnig the engine
+local resourceList = engine.package(patterns);
+
+resourceList
+

templates/engine/aks-k8s.jsonnet

@@ -0,0 +1,45 @@
+
+local k8s = import "k8s.jsonnet";
+
+local ns = {
+    apiVersion: "v1",
+    kind: "Namespace",
+    metadata: {
+        name: "trustgraph",
+    },
+    "spec": {
+    },
+};
+
+local sc = {
+    apiVersion: "storage.k8s.io/v1",
+    kind: "StorageClass",
+    metadata: {
+        name: "tg",
+    },
+    provisioner: "disk.csi.azure.com",
+    parameters: {
+        // Standard disks (spinning magnetic), Locally Redundant Storage
+        // Cheapest, basically
+        skuName: "Standard_LRS",
+    },
+    reclaimPolicy: "Delete",
+    volumeBindingMode: "WaitForFirstConsumer",
+};
+
+k8s + {
+
+    // Extract resources usnig the engine
+    package:: function(patterns)
+        local resources = [sc, ns] + std.flattenArrays([
+            p.create(self) for p in std.objectValues(patterns)
+        ]);
+        local resourceList = {
+            apiVersion: "v1",
+            kind: "List",
+            items: [ns, sc] + resources,
+        };
+        resourceList
+
+}
+

templates/engine/docker-compose.jsonnet

@@ -22,6 +22,8 @@
 
         with_image:: function(x) self + { image: x },
 
+        with_user:: function(x) self + { user: x },
+
         with_command:: function(x) self + { command: x },
 
         with_environment:: function(x) self + {
@@ -75,6 +77,10 @@
                 { command: container.command }
                 else {}) +
 
+                (if std.objectHas(container, "user") then
+                { user: container.user }
+                else {}) +
+
                 (if ! std.isEmpty(container.environment) then
                 { environment: container.environment }
                 else {}) +

templates/engine/k8s.jsonnet

@@ -14,6 +14,8 @@
 
         with_image:: function(x) self + { image: x },
 
+        with_user:: function(x) self + { user: x },
+
         with_command:: function(x) self + { command: x },
 
         with_environment:: function(x) self + {

templates/generate

@@ -7,6 +7,8 @@ import logging
 import os
 import sys
 import zipfile
+import pathlib
+from io import BytesIO
 
 logger = logging.getLogger("generate")
 logging.basicConfig(level=logging.INFO, format='%(message)s')
@@ -15,9 +17,13 @@ private_json = "Put your GCP private.json here"
 
 class Generator:
 
-    def __init__(self, config, base="./templates/", version="0.0.0"):
+    def __init__(
+        self, config, templates="./templates/", resources="./resources",
+        version="0.0.0",
+    ):
 
-        self.jsonnet_base = base
+        self.templates = pathlib.Path(templates)
+        self.resources = pathlib.Path(resources)
         self.config = config
         self.version = f"\"{version}\"".encode("utf-8")
 
@@ -34,25 +40,30 @@ class Generator:
             path = os.path.join(".", dir, filename)
             return str(path), self.config
 
-        if filename == "version.jsonnet" and dir == "./templates/values/":
+        if filename == "version.jsonnet" and dir == "templates/values/":
             path = os.path.join(".", dir, filename)
             return str(path), self.version
 
         if dir:
             candidates = [
-                os.path.join(".", dir, filename),
+                self.templates.joinpath(dir, filename),
-                os.path.join(".", filename)
+                self.templates.joinpath(filename),
+                self.resources.joinpath(dir, filename),
+                self.resources.joinpath(filename),
+                pathlib.Path(dir).joinpath(filename),
             ]
         else:
             candidates = [
-                os.path.join(".", filename)
+                self.templates.joinpath(filename),
+                pathlib.Path(dir).joinpath(filename),
+                pathlib.Path(filename),
             ]
 
         try:
 
             if filename == "vertexai/private.json":
 
-                return candidates[0], private_json.encode("utf-8")
+                return str(candidates[0]), private_json.encode("utf-8")
 
             for c in candidates:
                 logger.debug("Try: %s", c)
@@ -68,73 +79,157 @@ class Generator:
                 
         except:
 
-            path = os.path.join(self.jsonnet_base, filename)
+            path = os.path.join(self.templates, filename)
             logger.debug("Try: %s", path)
             with open(path, "rb") as f:
                 logger.debug("Loaded: %s", path)
                 return str(path), f.read()
 
+class Packager:
+
+    def __init__(self):
+        self.templates = pathlib.Path("./templates")
+        self.resources = pathlib.Path("./")
+
+    def process(
+            self, config, version="0.0.0", platform="docker-compose",
+    ):
+
+        config = config.encode("utf-8")
+
+        gen = Generator(
+            config, templates=self.templates, resources=self.resources,
+            version=version
+        )
+
+        path = self.templates.joinpath(
+            f"config-to-{platform}.jsonnet"
+        )
+        wrapper = path.read_text()
+
+        processed = gen.process(wrapper)
+
+        return processed
+    
+    def generate(self, config, version, platform):
+
+        logger.info(f"Generating for platform={platform} version={version}")
+
+        try:
+
+            if platform in set(["docker-compose", "podman-compose"]):
+                return self.generate_docker_compose(
+                    "docker-compose", version, config
+                )
+            elif platform in set(["minikube-k8s", "gcp-k8s", "aks-k8s"]):
+                return self.generate_k8s(
+                    platform, version, config
+                )
+            else:
+                raise RuntimeError("Bad configuration")
+
+        except Exception as e:
+            logging.error(f"Exception: {e}")
+            raise e
+
+    def generate_docker_compose(self, platform, version, config):
+
+        processed = self.process(
+            config, platform=platform, version=version
+        )
+
+        y = yaml.dump(processed)
+
+        mem = BytesIO()
+
+        with zipfile.ZipFile(mem, mode='w') as out:
+
+            def output(name, content):
+                logger.info(f"Adding {name}...")
+                out.writestr(name, content)
+
+            fname = "docker-compose.yaml"
+
+            output(fname, y)
+
+            # Grafana config
+            path = self.resources.joinpath(
+                "grafana/dashboards/dashboard.json"
+            )
+            res = path.read_text()
+            output("grafana/dashboards/dashboard.json", res)
+
+            path = self.resources.joinpath(
+                "grafana/provisioning/dashboard.yml"
+            )
+            res = path.read_text()
+            output("grafana/provisioning/dashboard.yml", res)
+
+            path = self.resources.joinpath(
+                "grafana/provisioning/datasource.yml"
+            )
+            res = path.read_text()
+            output("grafana/provisioning/datasource.yml", res)
+
+            # Prometheus config
+            path = self.resources.joinpath(
+                "prometheus/prometheus.yml"
+            )
+            res = path.read_text()
+            output("prometheus/prometheus.yml", res)
+
+        logger.info("Generation complete.")
+
+        return mem.getvalue()
+
+    def generate_k8s(self, platform, version, config):
+
+        processed = self.process(
+            config, platform=platform, version=version
+        )
+
+        y = yaml.dump(processed)
+
+        mem = BytesIO()
+
+        with zipfile.ZipFile(mem, mode='w') as out:
+
+            def output(name, content):
+                logger.info(f"Adding {name}...")
+                out.writestr(name, content)
+
+            fname = "resources.yaml"
+
+            output(fname, y)
+
+        logger.info("Generation complete.")
+
+        return mem.getvalue()
+            
 def main():
 
-    if len(sys.argv) != 3:
+    if len(sys.argv) != 4:
         print()
         print("Usage:")
-        print("  generate <outfile> <version> < input.json")
+        print("  generate <outfile> <version> <platform> < input.json")
         print()
         sys.exit(1)
 
     outfile = sys.argv[1]
     version = sys.argv[2]
+    platform = sys.argv[3]
 
     cfg = sys.stdin.read()
-    cfg = json.loads(cfg)
     
     logger.info(f"Outputting to {outfile}...")
 
-    with zipfile.ZipFile(outfile, mode='w') as out:
+    p = Packager()
+    resp = p.generate(cfg, version, platform)
 
-        def output(name, content):
+    with open(outfile, "wb") as f:
-            logger.info(f"Adding {name}...")
+        f.write(resp)
-            out.writestr(name, content)
 
-        fname = "tg-launch.yaml"
+    return
-
-        platform = "docker-compose"
-
-        with open(f"./templates/config-to-{platform}.jsonnet", "r") as f:
-            wrapper = f.read()
-
-        gen = Generator(json.dumps(cfg).encode("utf-8"), version=version)
-
-        processed = gen.process(wrapper)
-
-        y = yaml.dump(processed)
-
-        output(fname, y)
-
-        # Placeholder for the private.json file.  Won't put actual credentials
-        # here.
-        output("docker-compose/vertexai/private.json", private_json)
-
-        # Grafana config
-        with open("grafana/dashboards/dashboard.json") as f:
-            output(
-                "docker-compose/grafana/dashboards/dashboard.json", f.read()
-            )
-
-        with open("grafana/provisioning/dashboard.yml") as f:
-            output(
-                "docker-compose/grafana/provisioning/dashboard.yml", f.read()
-            )
-
-        with open("grafana/provisioning/datasource.yml") as f:
-            output(
-                "docker-compose/grafana/provisioning/datasource.yml", f.read()
-            )
-
-        # Prometheus config
-        with open("prometheus/prometheus.yml") as f:
-            output("docker-compose/prometheus/prometheus.yml", f.read())
 
 main()
 

templates/generate-all

@@ -87,8 +87,18 @@ def full_config_object(
 ):
 
     return config_object([
-        graph_store, "pulsar", vector_store, embeddings,
+        "triple-store-" + graph_store,
-        "graph-rag", "grafana", "trustgraph", llm, "workbench-ui",
+        "pulsar",
+        "vector-store-" + vector_store,
+        embeddings,
+        "graph-rag",
+        "grafana",
+        "trustgraph-base",
+        llm,
+        llm + "-rag",
+        "workbench-ui",
+        "prompt-template",
+        "agent-manager-react",
     ])
 
 def generate_config(
@@ -124,7 +134,7 @@ def generate_all(output, version):
     ]:
         for model in [
                 # "azure", "azure-openai", "bedrock", "claude", "cohere",
-                # "googleaistudio", "llamafile",
+                # "googleaistudio", "llamafile", "mistral",
                 "ollama",
                 # "openai", "vertexai",
         ]:

templates/patterns/llm-mistral.jsonnet

@@ -0,0 +1,32 @@
+{
+    pattern: {
+	name: "mistral",
+        icon: "🤖💬",
+        title: "Add Mistral LLM endpoint for text completion",
+	description: "This pattern integrates a Mistral LLM service for text completion operations.  You need a Mistral subscription and have an API key to be able to use this service.",
+        requires: ["pulsar", "trustgraph"],
+        features: ["llm"],
+	args: [
+	    {
+		name: "mistral-max-output-tokens",
+                label: "Maximum output tokens",
+		type: "integer",
+		description: "Limit on number tokens to generate",
+                default: 4096,
+		required: true,
+            },
+	    {
+		name: "mistral-temperature",
+                label: "Temperature",
+		type: "slider",
+		description: "Controlling predictability / creativity balance",
+                min: 0,
+                max: 1,
+                step: 0.05,
+                default: 0.5,
+            },
+	],
+        category: [ "llm" ],
+    },
+    module: "components/mistral.jsonnet",
+}

templates/stores/cassandra.jsonnet

@@ -13,7 +13,7 @@ local images = import "values/images.jsonnet";
                 engine.container("cassandra")
                     .with_image(images.cassandra)
                     .with_environment({
-                        JVM_OPTS: "-Xms300M -Xmx300M",
+                        JVM_OPTS: "-Xms300M -Xmx300M -Dcassandra.skip_wait_for_gossip_to_settle=0",
                     })
                     .with_limits("1.0", "1000M")
                     .with_reservations("0.5", "1000M")

templates/stores/memgraph.jsonnet

@@ -7,6 +7,8 @@ local images = import "values/images.jsonnet";
     
         create:: function(engine)
 
+            local vol = engine.volume("memgraph").with_size("20G");
+
             local container =
                 engine.container("memgraph")
                     .with_image(images.memgraph_mage)
@@ -16,7 +18,8 @@ local images = import "values/images.jsonnet";
                     .with_limits("1.0", "1000M")
                     .with_reservations("0.5", "1000M")
                     .with_port(7474, 7474, "api")
-                    .with_port(7687, 7687, "api2");
+                    .with_port(7687, 7687, "api2")
+                    .with_volume_mount(vol, "/var/lib/memgraph");
 
             local containerSet = engine.containers(
                 "memgraph", [ container ]
@@ -28,6 +31,7 @@ local images = import "values/images.jsonnet";
                 .with_port(7687, 7687, "api2");
 
             engine.resources([
+                vol,
                 containerSet,
                 service,
             ])
@@ -65,4 +69,3 @@ local images = import "values/images.jsonnet";
     },
 
 }
-

templates/stores/milvus.jsonnet

@@ -1,7 +1,8 @@
 local base = import "base/base.jsonnet";
 local images = import "values/images.jsonnet";
+local minio = import "stores/minio.jsonnet";
 
-{
+minio {
 
     etcd +: {
     
@@ -47,47 +48,6 @@ local images = import "values/images.jsonnet";
 
     },
 
-    mino +: {
-    
-        create:: function(engine)
-
-            local vol = engine.volume("minio-data").with_size("20G");
-
-            local container =
-                engine.container("minio")
-                    .with_image(images.minio)
-                    .with_command([
-                        "minio",
-                        "server",
-                        "/minio_data",
-                        "--console-address",
-                        ":9001",
-                    ])
-                    .with_environment({
-                        MINIO_ROOT_USER: "minioadmin",
-                        MINIO_ROOT_PASSWORD: "minioadmin",
-                    })
-                    .with_limits("0.5", "128M")
-                    .with_reservations("0.25", "128M")
-                    .with_port(9001, 9001, "api")
-                    .with_volume_mount(vol, "/minio_data");
-
-            local containerSet = engine.containers(
-                "etcd", [ container ]
-            );
-
-            local service =
-                engine.service(containerSet)
-                .with_port(9001, 9001, "api");
-
-            engine.resources([
-                vol,
-                containerSet,
-                service,
-            ])
-
-    },
-
     milvus +: {
     
         create:: function(engine)

templates/stores/minio.jsonnet

@@ -0,0 +1,49 @@
+local base = import "base/base.jsonnet";
+local images = import "values/images.jsonnet";
+
+{
+
+    minio +: {
+    
+        create:: function(engine)
+
+            local vol = engine.volume("minio-data").with_size("20G");
+
+            local container =
+                engine.container("minio")
+                    .with_image(images.minio)
+                    .with_command([
+                        "minio",
+                        "server",
+                        "/minio_data",
+                        "--console-address",
+                        ":9001",
+                    ])
+                    .with_environment({
+                        MINIO_ROOT_USER: "minioadmin",
+                        MINIO_ROOT_PASSWORD: "minioadmin",
+                    })
+                    .with_limits("0.5", "128M")
+                    .with_reservations("0.25", "128M")
+                    .with_port(9000, 9000, "api")
+                    .with_port(9001, 9001, "console")
+                    .with_volume_mount(vol, "/minio_data");
+
+            local containerSet = engine.containers(
+                "etcd", [ container ]
+            );
+
+            local service =
+                engine.service(containerSet)
+                .with_port(9000, 9000, "api")
+                .with_port(9001, 9001, "console");
+
+            engine.resources([
+                vol,
+                containerSet,
+                service,
+            ])
+
+    },
+
+}

templates/stores/neo4j.jsonnet

@@ -14,12 +14,14 @@ local images = import "values/images.jsonnet";
                     .with_image(images.neo4j)
                     .with_environment({
                         NEO4J_AUTH: "neo4j/password",
+                        NEO4J_server_memory_pagecache_size: "512m",
+                        NEO4J_server_memory_heap_max__size: "512m",
         //		NEO4J_server_bolt_listen__address: "0.0.0.0:7687",
         //		NEO4J_server_default__listen__address: "0.0.0.0",
         //		NEO4J_server_http_listen__address: "0.0.0.0:7474",
                     })
-                    .with_limits("1.0", "768M")
+                    .with_limits("1.0", "1536M")
-                    .with_reservations("0.5", "768M")
+                    .with_reservations("0.5", "1536M")
                     .with_port(7474, 7474, "api")
                     .with_port(7687, 7687, "api2")
                     .with_volume_mount(vol, "/data");

templates/values/images.jsonnet

@@ -1,18 +1,23 @@
 local version = import "version.jsonnet";
 {
     cassandra: "docker.io/cassandra:4.1.6",
-    neo4j: "docker.io/neo4j:5.22.0-community-bullseye",
+    neo4j: "docker.io/neo4j:5.26.0-community-bullseye",
     pulsar: "docker.io/apachepulsar/pulsar:3.3.1",
     pulsar_manager: "docker.io/apachepulsar/pulsar-manager:v0.4.0",
     etcd: "quay.io/coreos/etcd:v3.5.15",
-    minio: "docker.io/minio/minio:RELEASE.2024-08-17T01-24-54Z",
+    minio: "docker.io/minio/minio:RELEASE.2025-02-03T21-03-04Z",
     milvus: "docker.io/milvusdb/milvus:v2.4.9",
     prometheus: "docker.io/prom/prometheus:v2.53.2",
     grafana: "docker.io/grafana/grafana:11.1.4",
-    trustgraph: "docker.io/trustgraph/trustgraph-flow:" + version,
+    trustgraph_base: "docker.io/trustgraph/trustgraph-base:" + version,
-    qdrant: "docker.io/qdrant/qdrant:v1.11.1",
+    trustgraph_flow: "docker.io/trustgraph/trustgraph-flow:" + version,
+    trustgraph_ocr: "docker.io/trustgraph/trustgraph-ocr:" + version,
+    trustgraph_bedrock: "docker.io/trustgraph/trustgraph-bedrock:" + version,
+    trustgraph_vertexai: "docker.io/trustgraph/trustgraph-vertexai:" + version,
+    trustgraph_hf: "docker.io/trustgraph/trustgraph-hf:" + version,
+    qdrant: "docker.io/qdrant/qdrant:v1.13.3",
     memgraph_mage: "docker.io/memgraph/memgraph-mage:1.22-memgraph-2.22",
     memgraph_lab: "docker.io/memgraph/lab:2.19.1",
     falkordb: "docker.io/falkordb/falkordb:latest",
-    "workbench-ui": "docker.io/trustgraph/workbench-ui:0.1.6",
+    "workbench-ui": "docker.io/trustgraph/workbench-ui:0.2.4",
 }

trustgraph-base/trustgraph/api/api.py

@@ -102,11 +102,21 @@ class Api:
         except:
             raise ProtocolException(f"Response not formatted correctly")
 
-    def graph_rag(self, question):
+    def graph_rag(
+            self, question, user="trustgraph", collection="default",
+            entity_limit=50, triple_limit=30, max_subgraph_size=150,
+            max_path_length=2,
+    ):
 
         # The input consists of a question
         input = {
-            "query": question
+            "query": question,
+            "user": user,
+            "collection": collection,
+            "entity-limit": entity_limit,
+            "triple-limit": triple_limit,
+            "max-subgraph-size": max_subgraph_size,
+            "max-path-length": max_path_length,
         }
 
         url = f"{self.url}graph-rag"
@@ -131,6 +141,41 @@ class Api:
         except:
             raise ProtocolException(f"Response not formatted correctly")
 
+    def document_rag(
+            self, question, user="trustgraph", collection="default",
+            doc_limit=10,
+    ):
+
+        # The input consists of a question
+        input = {
+            "query": question,
+            "user": user,
+            "collection": collection,
+            "doc-limit": doc_limit,
+        }
+
+        url = f"{self.url}document-rag"
+
+        # Invoke the API, input is passed as JSON
+        resp = requests.post(url, json=input)
+
+        # Should be a 200 status code
+        if resp.status_code != 200:
+            raise ProtocolException(f"Status code {resp.status_code}")
+
+        try:
+            # Parse the response as JSON
+            object = resp.json()
+        except:
+            raise ProtocolException(f"Expected JSON response")
+
+        self.check_error(resp)
+
+        try:
+            return object["response"]
+        except:
+            raise ProtocolException(f"Response not formatted correctly")
+
     def embeddings(self, text):
 
         # The input consists of a text block

trustgraph-base/trustgraph/base/__init__.py

@@ -3,4 +3,6 @@ from . base_processor import BaseProcessor
 from . consumer import Consumer
 from . producer import Producer
 from . consumer_producer import ConsumerProducer
+from . publisher import Publisher
+from . subscriber import Subscriber
 

trustgraph-base/trustgraph/base/base_processor.py

@@ -1,4 +1,5 @@
 
+import asyncio
 import os
 import argparse
 import pulsar
@@ -11,6 +12,7 @@ from .. log_level import LogLevel
 class BaseProcessor:
 
     default_pulsar_host = os.getenv("PULSAR_HOST", 'pulsar://pulsar:6650')
+    default_pulsar_api_key = os.getenv("PULSAR_API_KEY", None)
 
     def __init__(self, **params):
 
@@ -28,14 +30,28 @@ class BaseProcessor:
         })
 
         pulsar_host = params.get("pulsar_host", self.default_pulsar_host)
+        pulsar_listener = params.get("pulsar_listener", None)
+        pulsar_api_key = params.get("pulsar_api_key", None)
         log_level = params.get("log_level", LogLevel.INFO)
 
         self.pulsar_host = pulsar_host
+        self.pulsar_api_key = pulsar_api_key
 
-        self.client = pulsar.Client(
+        if pulsar_api_key:
+            auth = pulsar.AuthenticationToken(pulsar_api_key)
+            self.client = pulsar.Client(
             pulsar_host,
+            authentication=auth,
             logger=pulsar.ConsoleLogger(log_level.to_pulsar())
-        )
+            )
+        else:
+            self.client = pulsar.Client(
+            pulsar_host,
+            listener_name=pulsar_listener,
+            logger=pulsar.ConsoleLogger(log_level.to_pulsar())
+            )
+
+        self.pulsar_listener = pulsar_listener
 
     def __del__(self):
 
@@ -52,6 +68,17 @@ class BaseProcessor:
             help=f'Pulsar host (default: {__class__.default_pulsar_host})',
         )
         
+        parser.add_argument(
+            '--pulsar-api-key',
+            default=__class__.default_pulsar_api_key,
+            help=f'Pulsar API key',
+        )
+
+        parser.add_argument(
+            '--pulsar-listener',
+            help=f'Pulsar listener (default: none)',
+        )
+
         parser.add_argument(
             '-l', '--log-level',
             type=LogLevel,
@@ -74,11 +101,20 @@ class BaseProcessor:
             help=f'Pulsar host (default: 8000)',
         )
 
-    def run(self):
+    async def start(self):
+        pass
+
+    async def run(self):
         raise RuntimeError("Something should have implemented the run method")
 
     @classmethod
-    def start(cls, prog, doc):
+    async def launch_async(cls, args):
+        p = cls(**args)
+        await p.start()
+        await p.run()
+
+    @classmethod
+    def launch(cls, prog, doc):
 
         parser = argparse.ArgumentParser(
             prog=prog,
@@ -99,8 +135,7 @@ class BaseProcessor:
 
             try:
 
-                p = cls(**args)
+                asyncio.run(cls.launch_async(args))
-                p.run()
 
             except KeyboardInterrupt:
                 print("Keyboard interrupt.")
@@ -118,3 +153,4 @@ class BaseProcessor:
                 print("Will retry...", flush=True)
 
                 time.sleep(4)
+

trustgraph-base/trustgraph/base/consumer.py

@@ -1,11 +1,16 @@
 
+import asyncio
 from pulsar.schema import JsonSchema
+import pulsar
 from prometheus_client import Histogram, Info, Counter, Enum
 import time
 
 from . base_processor import BaseProcessor
 from .. exceptions import TooManyRequests
 
+default_rate_limit_retry = 10
+default_rate_limit_timeout = 7200
+
 class Consumer(BaseProcessor):
 
     def __init__(self, **params):
@@ -21,11 +26,18 @@ class Consumer(BaseProcessor):
 
         super(Consumer, self).__init__(**params)
 
-        input_queue = params.get("input_queue")
+        self.input_queue = params.get("input_queue")
-        subscriber = params.get("subscriber")
+        self.subscriber = params.get("subscriber")
-        input_schema = params.get("input_schema")
+        self.input_schema = params.get("input_schema")
 
-        if input_schema == None:
+        self.rate_limit_retry = params.get(
+            "rate_limit_retry", default_rate_limit_retry
+        )
+        self.rate_limit_timeout = params.get(
+            "rate_limit_timeout", default_rate_limit_timeout
+        )
+
+        if self.input_schema == None:
             raise RuntimeError("input_schema must be specified")
 
         if not hasattr(__class__, "request_metric"):
@@ -43,18 +55,28 @@ class Consumer(BaseProcessor):
                 'processing_count', 'Processing count', ["status"]
             )
 
+        if not hasattr(__class__, "rate_limit_metric"):
+            __class__.rate_limit_metric = Counter(
+                'rate_limit_count', 'Rate limit event count',
+            )
+
         __class__.pubsub_metric.info({
-            "input_queue": input_queue,
+            "input_queue": self.input_queue,
-            "subscriber": subscriber,
+            "subscriber": self.subscriber,
-            "input_schema": input_schema.__name__,
+            "input_schema": self.input_schema.__name__,
+            "rate_limit_retry": str(self.rate_limit_retry),
+            "rate_limit_timeout": str(self.rate_limit_timeout),
         })
 
         self.consumer = self.client.subscribe(
-            input_queue, subscriber,
+            self.input_queue, self.subscriber,
-            schema=JsonSchema(input_schema),
+            consumer_type=pulsar.ConsumerType.Shared,
+            schema=JsonSchema(self.input_schema),
         )
 
-    def run(self):
+        print("Initialised consumer.", flush=True)
+
+    async def run(self):
 
         __class__.state_metric.state('running')
 
@@ -62,31 +84,61 @@ class Consumer(BaseProcessor):
 
             msg = self.consumer.receive()
 
-            try:
+            expiry = time.time() + self.rate_limit_timeout
 
-                with __class__.request_metric.time():
+            # This loop is for retry on rate-limit / resource limits
-                    self.handle(msg)
+            while True:
 
-                # Acknowledge successful processing of the message
+                if time.time() > expiry:
-                self.consumer.acknowledge(msg)
 
-                __class__.processing_metric.labels(status="success").inc()
+                    print("Gave up waiting for rate-limit retry", flush=True)
 
-            except TooManyRequests:
+                    # Message failed to be processed, this causes it to
-                self.consumer.negative_acknowledge(msg)
+                    # be retried
-                print("TooManyRequests: will retry")
+                    self.consumer.negative_acknowledge(msg)
-                __class__.processing_metric.labels(status="rate-limit").inc()
-                time.sleep(5)
-                continue
 
-            except Exception as e:
+                    __class__.processing_metric.labels(status="error").inc()
 
-                print("Exception:", e, flush=True)
+                    # Break out of retry loop, processes next message
+                    break
 
-                # Message failed to be processed
+                try:
-                self.consumer.negative_acknowledge(msg)
 
-                __class__.processing_metric.labels(status="error").inc()
+                    with __class__.request_metric.time():
+                        await self.handle(msg)
+
+                    # Acknowledge successful processing of the message
+                    self.consumer.acknowledge(msg)
+
+                    __class__.processing_metric.labels(status="success").inc()
+
+                    # Break out of retry loop
+                    break
+
+                except TooManyRequests:
+
+                    print("TooManyRequests: will retry...", flush=True)
+
+                    __class__.rate_limit_metric.inc()
+
+                    # Sleep
+                    time.sleep(self.rate_limit_retry)
+
+                    # Contine from retry loop, just causes a reprocessing
+                    continue
+                
+                except Exception as e:
+
+                    print("Exception:", e, flush=True)
+
+                    # Message failed to be processed, this causes it to
+                    # be retried
+                    self.consumer.negative_acknowledge(msg)
+
+                    __class__.processing_metric.labels(status="error").inc()
+
+                    # Break out of retry loop, processes next message
+                    break
 
     @staticmethod
     def add_args(parser, default_input_queue, default_subscriber):
@@ -105,3 +157,17 @@ class Consumer(BaseProcessor):
             help=f'Queue subscriber name (default: {default_subscriber})'
         )
 
+        parser.add_argument(
+            '--rate-limit-retry',
+            type=int,
+            default=default_rate_limit_retry,
+            help=f'Rate limit retry (default: {default_rate_limit_retry})'
+        )
+
+        parser.add_argument(
+            '--rate-limit-timeout',
+            type=int,
+            default=default_rate_limit_timeout,
+            help=f'Rate limit timeout (default: {default_rate_limit_timeout})'
+        )
+

trustgraph-base/trustgraph/base/consumer_producer.py

@@ -1,113 +1,48 @@
 
 from pulsar.schema import JsonSchema
+import pulsar
 from prometheus_client import Histogram, Info, Counter, Enum
 import time
 
-from . base_processor import BaseProcessor
+from . consumer import Consumer
 from .. exceptions import TooManyRequests
 
-# FIXME: Derive from consumer?  And producer?
+class ConsumerProducer(Consumer):
-
-class ConsumerProducer(BaseProcessor):
 
     def __init__(self, **params):
 
-        if not hasattr(__class__, "state_metric"):
+        super(ConsumerProducer, self).__init__(**params)
-            __class__.state_metric = Enum(
-                'processor_state', 'Processor state',
-                states=['starting', 'running', 'stopped']
-            )
-            __class__.state_metric.state('starting')
 
-        __class__.state_metric.state('starting')
+        self.output_queue = params.get("output_queue")
-
+        self.output_schema = params.get("output_schema")
-        input_queue = params.get("input_queue")
-        output_queue = params.get("output_queue")
-        subscriber = params.get("subscriber")
-        input_schema = params.get("input_schema")
-        output_schema = params.get("output_schema")
-
-        if not hasattr(__class__, "request_metric"):
-            __class__.request_metric = Histogram(
-                'request_latency', 'Request latency (seconds)'
-            )
 
         if not hasattr(__class__, "output_metric"):
             __class__.output_metric = Counter(
                 'output_count', 'Output items created'
             )
 
-        if not hasattr(__class__, "pubsub_metric"):
-            __class__.pubsub_metric = Info(
-                'pubsub', 'Pub/sub configuration'
-            )
-
-        if not hasattr(__class__, "processing_metric"):
-            __class__.processing_metric = Counter(
-                'processing_count', 'Processing count', ["status"]
-            )
-
         __class__.pubsub_metric.info({
-            "input_queue": input_queue,
+            "input_queue": self.input_queue,
-            "output_queue": output_queue,
+            "output_queue": self.output_queue,
-            "subscriber": subscriber,
+            "subscriber": self.subscriber,
-            "input_schema": input_schema.__name__,
+            "input_schema": self.input_schema.__name__,
-            "output_schema": output_schema.__name__,
+            "output_schema": self.output_schema.__name__,
+            "rate_limit_retry": str(self.rate_limit_retry),
+            "rate_limit_timeout": str(self.rate_limit_timeout),
         })
 
-        super(ConsumerProducer, self).__init__(**params)
+        if self.output_schema == None:
-
-        if input_schema == None:
-            raise RuntimeError("input_schema must be specified")
-
-        if output_schema == None:
             raise RuntimeError("output_schema must be specified")
 
         self.producer = self.client.create_producer(
-            topic=output_queue,
+            topic=self.output_queue,
-            schema=JsonSchema(output_schema),
+            schema=JsonSchema(self.output_schema),
+            chunking_enabled=True,
         )
 
-        self.consumer = self.client.subscribe(
+        print("Initialised consumer/producer.")
-            input_queue, subscriber,
-            schema=JsonSchema(input_schema),
-        )
 
-    def run(self):
+    async def send(self, msg, properties={}):
-
-        __class__.state_metric.state('running')
-
-        while True:
-
-            msg = self.consumer.receive()
-
-            try:
-
-                with __class__.request_metric.time():
-                    resp = self.handle(msg)
-
-                # Acknowledge successful processing of the message
-                self.consumer.acknowledge(msg)
-
-                __class__.processing_metric.labels(status="success").inc()
-
-            except TooManyRequests:
-                self.consumer.negative_acknowledge(msg)
-                print("TooManyRequests: will retry")
-                __class__.processing_metric.labels(status="rate-limit").inc()
-                time.sleep(5)
-                continue
-
-            except Exception as e:
-
-                print("Exception:", e, flush=True)
-
-                # Message failed to be processed
-                self.consumer.negative_acknowledge(msg)
-
-                __class__.processing_metric.labels(status="error").inc()
-
-    def send(self, msg, properties={}):
         self.producer.send(msg, properties)
         __class__.output_metric.inc()
 
@@ -117,19 +52,7 @@ class ConsumerProducer(BaseProcessor):
             default_output_queue,
     ):
 
-        BaseProcessor.add_args(parser)
+        Consumer.add_args(parser, default_input_queue, default_subscriber)
-
-        parser.add_argument(
-            '-i', '--input-queue',
-            default=default_input_queue,
-            help=f'Input queue (default: {default_input_queue})'
-        )
-
-        parser.add_argument(
-            '-s', '--subscriber',
-            default=default_subscriber,
-            help=f'Queue subscriber name (default: {default_subscriber})'
-        )
 
         parser.add_argument(
             '-o', '--output-queue',

trustgraph-base/trustgraph/base/producer.py

@@ -34,9 +34,10 @@ class Producer(BaseProcessor):
         self.producer = self.client.create_producer(
             topic=output_queue,
             schema=JsonSchema(output_schema),
+            chunking_enabled=True,
         )
 
-    def send(self, msg, properties={}):
+    async def send(self, msg, properties={}):
         self.producer.send(msg, properties)
         __class__.output_metric.inc()
 

trustgraph-base/trustgraph/base/publisher.py

@@ -6,37 +6,43 @@ import threading
 
 class Publisher:
 
-    def __init__(self, pulsar_host, topic, schema=None, max_size=10,
+    def __init__(self, pulsar_client, topic, schema=None, max_size=10,
-                 chunking_enabled=False):
+                 chunking_enabled=True):
-        self.pulsar_host = pulsar_host
+        self.client = pulsar_client
         self.topic = topic
         self.schema = schema
         self.q = queue.Queue(maxsize=max_size)
         self.chunking_enabled = chunking_enabled
+        self.running = True
 
     def start(self):
         self.task = threading.Thread(target=self.run)
         self.task.start()
 
+    def stop(self):
+        self.running = False
+
+    def join(self):
+        self.stop()
+        self.task.join()
+
     def run(self):
 
-        while True:
+        while self.running:
 
             try:
-
+                producer = self.client.create_producer(
-                client = pulsar.Client(
-                    self.pulsar_host,
-                )
-
-                producer = client.create_producer(
                     topic=self.topic,
                     schema=self.schema,
                     chunking_enabled=self.chunking_enabled,
                 )
 
-                while True:
+                while self.running:
 
-                    id, item = self.q.get()
+                    try:
+                        id, item = self.q.get(timeout=0.5)
+                    except queue.Empty:
+                        continue
 
                     if id:
                         producer.send(item, { "id": id })
@@ -51,3 +57,5 @@ class Publisher:
 
     def send(self, id, msg):
         self.q.put((id, msg))
+
+        

trustgraph-base/trustgraph/base/subscriber.py

@@ -6,9 +6,9 @@ import time
 
 class Subscriber:
 
-    def __init__(self, pulsar_host, topic, subscription, consumer_name,
+    def __init__(self, pulsar_client, topic, subscription, consumer_name,
                  schema=None, max_size=100):
-        self.pulsar_host = pulsar_host
+        self.client = pulsar_client
         self.topic = topic
         self.subscription = subscription
         self.consumer_name = consumer_name
@@ -17,29 +17,32 @@ class Subscriber:
         self.full = {}
         self.max_size = max_size
         self.lock = threading.Lock()
+        self.running = True
 
     def start(self):
         self.task = threading.Thread(target=self.run)
         self.task.start()
 
+    def stop(self):
+        self.running = False
+
+    def join(self):
+        self.task.join()
+
     def run(self):
 
-        while True:
+        while self.running:
 
             try:
 
-                client = pulsar.Client(
+                consumer = self.client.subscribe(
-                    self.pulsar_host,
-                )
-
-                consumer = client.subscribe(
                     topic=self.topic,
                     subscription_name=self.subscription,
                     consumer_name=self.consumer_name,
                     schema=self.schema,
                 )
 
-                while True:
+                while self.running:
 
                     msg = consumer.receive()
 
@@ -57,12 +60,14 @@ class Subscriber:
 
                         if id in self.q:
                             try:
+                                # FIXME: Timeout means data goes missing
                                 self.q[id].put(value, timeout=0.5)
                             except:
                                 pass
 
                         for q in self.full.values():
                             try:
+                                # FIXME: Timeout means data goes missing
                                 q.put(value, timeout=0.5)
                             except:
                                 pass

trustgraph-base/trustgraph/clients/agent_client.py

@@ -20,6 +20,7 @@ class AgentClient(BaseClient):
             input_queue=None,
             output_queue=None,
             pulsar_host="pulsar://pulsar:6650",
+            pulsar_api_key=None,
     ):
 
         if input_queue is None: input_queue = agent_request_queue
@@ -33,6 +34,7 @@ class AgentClient(BaseClient):
             pulsar_host=pulsar_host,
             input_schema=AgentRequest,
             output_schema=AgentResponse,
+            pulsar_api_key=pulsar_api_key
         )
 
     def request(

trustgraph-base/trustgraph/clients/base.py

@@ -27,6 +27,7 @@ class BaseClient:
             input_schema=None,
             output_schema=None,
             pulsar_host="pulsar://pulsar:6650",
+            pulsar_api_key=None,
     ):
 
         if input_queue == None: raise RuntimeError("Need input_queue")
@@ -37,10 +38,18 @@ class BaseClient:
         if subscriber == None:
             subscriber = str(uuid.uuid4())
 
-        self.client = pulsar.Client(
+        if pulsar_api_key:
+            auth = pulsar.AuthenticationToken(pulsar_api_key)
+            self.client = pulsar.Client(
             pulsar_host,
             logger=pulsar.ConsoleLogger(log_level),
-        )
+            authentication=auth,
+            )
+        else:
+            self.client = pulsar.Client(
+            pulsar_host,
+            logger=pulsar.ConsoleLogger(log_level)
+            )
 
         self.producer = self.client.create_producer(
             topic=input_queue,

trustgraph-base/trustgraph/clients/document_embeddings_client.py

@@ -20,6 +20,7 @@ class DocumentEmbeddingsClient(BaseClient):
             input_queue=None,
             output_queue=None,
             pulsar_host="pulsar://pulsar:6650",
+            pulsar_api_key=None,
     ):
 
         if input_queue == None:
@@ -34,12 +35,17 @@ class DocumentEmbeddingsClient(BaseClient):
             input_queue=input_queue,
             output_queue=output_queue,
             pulsar_host=pulsar_host,
+            pulsar_api_key=pulsar_api_key,
             input_schema=DocumentEmbeddingsRequest,
             output_schema=DocumentEmbeddingsResponse,
         )
 
-    def request(self, vectors, limit=10, timeout=300):
+    def request(
+            self, vectors, user="trustgraph", collection="default",
+            limit=10, timeout=300
+    ):
         return self.call(
+            user=user, collection=collection,
             vectors=vectors, limit=limit, timeout=timeout
         ).documents
 

trustgraph-base/trustgraph/clients/document_rag_client.py

@@ -20,6 +20,7 @@ class DocumentRagClient(BaseClient):
             input_queue=None,
             output_queue=None,
             pulsar_host="pulsar://pulsar:6650",
+            pulsar_api_key=None,
     ):
 
         if input_queue == None:
@@ -34,6 +35,7 @@ class DocumentRagClient(BaseClient):
             input_queue=input_queue,
             output_queue=output_queue,
             pulsar_host=pulsar_host,
+            pulsar_api_key=pulsar_api_key,
             input_schema=DocumentRagQuery,
             output_schema=DocumentRagResponse,
         )

trustgraph-base/trustgraph/clients/embeddings_client.py

@@ -20,6 +20,7 @@ class EmbeddingsClient(BaseClient):
             output_queue=None,
             subscriber=None,
             pulsar_host="pulsar://pulsar:6650",
+            pulsar_api_key=None,
     ):
 
         if input_queue == None:
@@ -34,6 +35,7 @@ class EmbeddingsClient(BaseClient):
             input_queue=input_queue,
             output_queue=output_queue,
             pulsar_host=pulsar_host,
+            pulsar_api_key=pulsar_api_key,
             input_schema=EmbeddingsRequest,
             output_schema=EmbeddingsResponse,
         )

trustgraph-base/trustgraph/clients/graph_embeddings_client.py

@@ -20,6 +20,7 @@ class GraphEmbeddingsClient(BaseClient):
             input_queue=None,
             output_queue=None,
             pulsar_host="pulsar://pulsar:6650",
+            pulsar_api_key=None,
     ):
 
         if input_queue == None:
@@ -34,6 +35,7 @@ class GraphEmbeddingsClient(BaseClient):
             input_queue=input_queue,
             output_queue=output_queue,
             pulsar_host=pulsar_host,
+            pulsar_api_key=pulsar_api_key,
             input_schema=GraphEmbeddingsRequest,
             output_schema=GraphEmbeddingsResponse,
         )

trustgraph-base/trustgraph/clients/graph_rag_client.py

@@ -20,6 +20,7 @@ class GraphRagClient(BaseClient):
             input_queue=None,
             output_queue=None,
             pulsar_host="pulsar://pulsar:6650",
+            pulsar_api_key=None,
     ):
 
         if input_queue == None:
@@ -34,6 +35,7 @@ class GraphRagClient(BaseClient):
             input_queue=input_queue,
             output_queue=output_queue,
             pulsar_host=pulsar_host,
+            pulsar_api_key=pulsar_api_key,
             input_schema=GraphRagQuery,
             output_schema=GraphRagResponse,
         )

trustgraph-base/trustgraph/clients/llm_client.py

@@ -20,6 +20,7 @@ class LlmClient(BaseClient):
             input_queue=None,
             output_queue=None,
             pulsar_host="pulsar://pulsar:6650",
+            pulsar_api_key=None,
     ):
 
         if input_queue is None: input_queue = text_completion_request_queue
@@ -31,6 +32,7 @@ class LlmClient(BaseClient):
             input_queue=input_queue,
             output_queue=output_queue,
             pulsar_host=pulsar_host,
+            pulsar_api_key=pulsar_api_key,
             input_schema=TextCompletionRequest,
             output_schema=TextCompletionResponse,
         )

trustgraph-base/trustgraph/clients/prompt_client.py

@@ -39,6 +39,7 @@ class PromptClient(BaseClient):
             input_queue=None,
             output_queue=None,
             pulsar_host="pulsar://pulsar:6650",
+            pulsar_api_key=None,
     ):
 
         if input_queue == None:
@@ -53,6 +54,7 @@ class PromptClient(BaseClient):
             input_queue=input_queue,
             output_queue=output_queue,
             pulsar_host=pulsar_host,
+            pulsar_api_key=pulsar_api_key,
             input_schema=PromptRequest,
             output_schema=PromptResponse,
         )

trustgraph-base/trustgraph/clients/triples_query_client.py

@@ -21,6 +21,7 @@ class TriplesQueryClient(BaseClient):
             input_queue=None,
             output_queue=None,
             pulsar_host="pulsar://pulsar:6650",
+            pulsar_api_key=None,
     ):
 
         if input_queue == None:
@@ -34,6 +35,7 @@ class TriplesQueryClient(BaseClient):
             subscriber=subscriber,
             input_queue=input_queue,
             output_queue=output_queue,
+            pulsar_api_key=pulsar_api_key,
             pulsar_host=pulsar_host,
             input_schema=TriplesQueryRequest,
             output_schema=TriplesQueryResponse,

trustgraph-base/trustgraph/exceptions.py

@@ -8,7 +8,6 @@ class LlmError(Exception):
 class ParseError(Exception):
     pass
 
-
+class RequestError(Exception):
-
+    pass
-
 

trustgraph-base/trustgraph/schema/__init__.py

@@ -10,5 +10,6 @@ from . retrieval import *
 from . metadata import *
 from . agent import *
 from . lookup import *
+from . library import *
 
 

trustgraph-base/trustgraph/schema/documents.py

@@ -35,14 +35,18 @@ chunk_ingest_queue = topic('chunk-load')
 
 ############################################################################
 
-# Chunk embeddings are an embeddings associated with a text chunk
+# Document embeddings are embeddings associated with a chunk
 
 class ChunkEmbeddings(Record):
-    metadata = Metadata()
-    vectors = Array(Array(Double()))
     chunk = Bytes()
+    vectors = Array(Array(Double()))
 
-chunk_embeddings_ingest_queue = topic('chunk-embeddings-load')
+# This is a 'batching' mechanism for the above data
+class DocumentEmbeddings(Record):
+    metadata = Metadata()
+    chunks = Array(ChunkEmbeddings())
+
+document_embeddings_store_queue = topic('document-embeddings-store')
 
 ############################################################################
 
@@ -51,6 +55,8 @@ chunk_embeddings_ingest_queue = topic('chunk-embeddings-load')
 class DocumentEmbeddingsRequest(Record):
     vectors = Array(Array(Double()))
     limit = Integer()
+    user = String()
+    collection = String()
 
 class DocumentEmbeddingsResponse(Record):
     error = Error()
@@ -62,3 +68,4 @@ document_embeddings_request_queue = topic(
 document_embeddings_response_queue = topic(
     'doc-embeddings', kind='non-persistent', namespace='response',
 )
+

trustgraph-base/trustgraph/schema/graph.py

@@ -7,12 +7,31 @@ from . metadata import Metadata
 
 ############################################################################
 
+# Entity context are an entity associated with textual context
+
+class EntityContext(Record):
+    entity = Value()
+    context = String()
+
+# This is a 'batching' mechanism for the above data
+class EntityContexts(Record):
+    metadata = Metadata()
+    entities = Array(EntityContext())
+
+entity_contexts_ingest_queue = topic('entity-contexts-load')
+
+############################################################################
+
 # Graph embeddings are embeddings associated with a graph entity
 
+class EntityEmbeddings(Record):
+    entity = Value()
+    vectors = Array(Array(Double()))
+
+# This is a 'batching' mechanism for the above data
 class GraphEmbeddings(Record):
     metadata = Metadata()
-    vectors = Array(Array(Double()))
+    entities = Array(EntityEmbeddings())
-    entity = Value()
 
 graph_embeddings_store_queue = topic('graph-embeddings-store')
 

trustgraph-base/trustgraph/schema/library.py

@@ -0,0 +1,74 @@
+
+from pulsar.schema import Record, Bytes, String, Array, Long
+from . types import Triple
+from . topic import topic
+from . types import Error
+from . metadata import Metadata
+from . documents import Document, TextDocument
+
+# add
+#   -> (id, document)
+#   <- ()
+#   <- (error)
+
+# list
+#   -> (user, collection?)
+#   <- (info)
+#   <- (error)
+
+# add(Metadata, Bytes) : error?
+# copy(id, user, collection)
+# move(id, user, collection)
+# delete(id)
+# get(id) : Bytes
+# reindex(id)
+# list(user, collection) : id[]
+# info(id[]) : DocumentInfo[]
+# search(<key,op,value>[]) : id[]
+
+class DocumentPackage(Record):
+    id = String()
+    document = Bytes()
+    kind = String()
+    user = String()
+    collection = String()
+    title = String()
+    comments = String()
+    time = Long()
+    metadata = Array(Triple())
+
+class DocumentInfo(Record):
+    id = String()
+    kind = String()
+    user = String()
+    collection = String()
+    title = String()
+    comments = String()
+    time = Long()
+    metadata = Array(Triple())
+
+class Criteria(Record):
+    key = String()
+    value = String()
+    operator = String()
+
+class LibrarianRequest(Record):
+    operation = String()
+    id = String()
+    document = DocumentPackage()
+    user = String()
+    collection = String()
+    criteria = Array(Criteria())
+
+class LibrarianResponse(Record):
+    error = Error()
+    document = DocumentPackage()
+    info = Array(DocumentInfo())
+
+librarian_request_queue = topic(
+    'librarian', kind='non-persistent', namespace='request'
+)
+librarian_response_queue = topic(
+    'librarian', kind='non-persistent', namespace='response',
+)
+

trustgraph-base/trustgraph/schema/retrieval.py

@@ -11,6 +11,10 @@ class GraphRagQuery(Record):
     query = String()
     user = String()
     collection = String()
+    entity_limit = Integer()
+    triple_limit = Integer()
+    max_subgraph_size = Integer()
+    max_path_length = Integer()
 
 class GraphRagResponse(Record):
     error = Error()
@@ -31,6 +35,7 @@ class DocumentRagQuery(Record):
     query = String()
     user = String()
     collection = String()
+    doc_limit = Integer()
 
 class DocumentRagResponse(Record):
     error = Error()

trustgraph-bedrock/setup.py

@@ -34,7 +34,7 @@ setuptools.setup(
     python_requires='>=3.8',
     download_url = "https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v" + version + ".tar.gz",
     install_requires=[
-        "trustgraph-base>=0.18,<0.19",
+        "trustgraph-base>=0.21,<0.22",
         "pulsar-client",
         "prometheus-client",
         "boto3",

trustgraph-bedrock/trustgraph/model/text_completion/bedrock/llm.py

@@ -8,6 +8,7 @@ import boto3
 import json
 from prometheus_client import Histogram
 import os
+import enum
 
 from .... schema import TextCompletionRequest, TextCompletionResponse, Error
 from .... schema import text_completion_request_queue
@@ -24,32 +25,163 @@ default_subscriber = module
 default_model = 'mistral.mistral-large-2407-v1:0'
 default_temperature = 0.0
 default_max_output = 2048
-default_aws_id_key = os.getenv("AWS_ID_KEY", None)
+default_top_p = 0.99
-default_aws_secret = os.getenv("AWS_SECRET", None)
+default_top_k = 40
-default_aws_region = os.getenv("AWS_REGION", 'us-west-2')
+
+# Actually, these could all just be None, no need to get environment
+# variables, as Boto3 would pick all these up if not passed in as args
+default_access_key_id = os.getenv("AWS_ACCESS_KEY_ID", None)
+default_secret_access_key = os.getenv("AWS_SECRET_ACCESS_KEY", None)
+default_session_token = os.getenv("AWS_SESSION_TOKEN", None)
+default_profile = os.getenv("AWS_PROFILE", None)
+default_region = os.getenv("AWS_DEFAULT_REGION", None)
+
+# Variant API handling depends on the model type
+
+class ModelHandler:
+    def __init__(self):
+        self.temperature = default_temperature
+        self.max_output = default_max_output
+        self.top_p = default_top_p
+        self.top_k = default_top_k
+    def set_temperature(self, temperature):
+        self.temperature = temperature
+    def set_max_output(self, max_output):
+        self.max_output = max_output
+    def set_top_p(self, top_p):
+        self.top_p = top_p
+    def set_top_k(self, top_k):
+        self.top_k = top_k
+    def encode_request(self, system, prompt):
+        raise RuntimeError("format_request not implemented")
+    def decode_response(self, response):
+        raise RuntimeError("format_request not implemented")
+
+class Mistral(ModelHandler):
+    def __init__(self):
+        self.top_p = 0.99
+        self.top_k = 40
+    def encode_request(self, system, prompt):
+        return json.dumps({
+            "prompt": f"{system}\n\n{prompt}",
+            "max_tokens": self.max_output,
+            "temperature": self.temperature,
+            "top_p": self.top_p,
+            "top_k": self.top_k,
+        })
+    def decode_response(self, response):
+        response_body = json.loads(response.get("body").read())
+        return response_body['outputs'][0]['text']
+
+# Llama 3
+class Meta(ModelHandler):
+    def __init__(self):
+        self.top_p = 0.95
+    def encode_request(self, system, prompt):
+        return json.dumps({
+            "prompt": f"{system}\n\n{prompt}",
+            "max_gen_len": self.max_output,
+            "temperature": self.temperature,
+            "top_p": self.top_p,
+        })
+    def decode_response(self, response):
+        model_response = json.loads(response["body"].read())
+        return model_response["generation"]
+
+class Anthropic(ModelHandler):
+    def __init__(self):
+        self.top_p = 0.999
+    def encode_request(self, system, prompt):
+        return json.dumps({
+            "anthropic_version": "bedrock-2023-05-31",
+            "max_tokens": self.max_output,
+            "temperature": self.temperature,
+            "top_p": self.top_p,
+            "messages": [
+                {
+                    "role": "user",
+                    "content": [
+                        {
+                            "type": "text",
+                            "text": f"{system}\n\n{prompt}",
+                        }
+                    ]
+                }
+            ]
+        })
+    def decode_response(self, response):
+        model_response = json.loads(response["body"].read())
+        return model_response['content'][0]['text']
+
+class Ai21(ModelHandler):
+    def __init__(self):
+        self.top_p = 0.9
+    def encode_request(self, system, prompt):
+        return json.dumps({
+            "max_tokens": self.max_output,
+            "temperature": self.temperature,
+            "top_p": self.top_p,
+            "messages": [
+                {
+                    "role": "user",
+                    "content": f"{system}\n\n{prompt}"
+                }
+            ]
+        })
+    def decode_response(self, response):
+        content = response['body'].read()
+        content_str = content.decode('utf-8')
+        content_json = json.loads(content_str)
+        return content_json['choices'][0]['message']['content']
+
+class Cohere(ModelHandler):
+    def encode_request(self, system, prompt):
+        return json.dumps({
+            "max_tokens": self.max_output,
+            "temperature": self.temperature,
+            "message": f"{system}\n\n{prompt}",
+        })
+    def decode_response(self, response):
+        content = response['body'].read()
+        content_str = content.decode('utf-8')
+        content_json = json.loads(content_str)
+        return content_json['text']
+
+Default=Mistral
 
 class Processor(ConsumerProducer):
 
     def __init__(self, **params):
 
+        print(params)
+    
         input_queue = params.get("input_queue", default_input_queue)
         output_queue = params.get("output_queue", default_output_queue)
         subscriber = params.get("subscriber", default_subscriber)
+
         model = params.get("model", default_model)
-        aws_id_key = params.get("aws_id_key", default_aws_id_key)
-        aws_secret = params.get("aws_secret", default_aws_secret)
-        aws_region = params.get("aws_region", default_aws_region)
         temperature = params.get("temperature", default_temperature)
         max_output = params.get("max_output", default_max_output)
 
-        if aws_id_key is None:
+        aws_access_key_id = params.get(
-            raise RuntimeError("AWS ID not specified")
+            "aws_access_key_id", default_access_key_id
+        )
 
-        if aws_secret is None:
+        aws_secret_access_key = params.get(
-            raise RuntimeError("AWS secret not specified")
+            "aws_secret_access_key", default_secret_access_key
+        )
 
-        if aws_region is None:
+        aws_session_token = params.get(
-            raise RuntimeError("AWS region not specified")
+            "aws_session_token", default_session_token
+        )
+
+        aws_region = params.get(
+            "aws_region", default_region
+        )
+
+        aws_profile = params.get(
+            "aws_profile", default_profile
+        )
 
         super(Processor, self).__init__(
             **params | {
@@ -81,17 +213,51 @@ class Processor(ConsumerProducer):
         self.temperature = temperature
         self.max_output = max_output
 
+        self.variant = self.determine_variant(self.model)()
+        self.variant.set_temperature(temperature)
+        self.variant.set_max_output(max_output)
+
         self.session = boto3.Session(
-            aws_access_key_id=aws_id_key,
+            aws_access_key_id=aws_access_key_id,
-            aws_secret_access_key=aws_secret,
+            aws_secret_access_key=aws_secret_access_key,
-            region_name=aws_region
+            aws_session_token=aws_session_token,
+            profile_name=aws_profile,
+            region_name=aws_region,
         )
 
         self.bedrock = self.session.client(service_name='bedrock-runtime')
 
         print("Initialised", flush=True)
 
-    def handle(self, msg):
+    def determine_variant(self, model):
+
+        # FIXME: Missing, Amazon models, Deepseek
+
+        # This set of conditions deals with normal bedrock on-demand usage
+        if self.model.startswith("mistral"):
+            return Mistral
+        elif self.model.startswith("meta"):
+            return Meta
+        elif self.model.startswith("anthropic"):
+            return Anthropic
+        elif self.model.startswith("ai21"):
+            return Ai21
+        elif self.model.startswith("cohere"):
+            return Cohere
+
+        # The inference profiles
+        if self.model.startswith("us.meta"):
+            return Meta
+        elif self.model.startswith("us.anthropic"):
+            return Anthropic
+        elif self.model.startswith("eu.meta"):
+            return Meta
+        elif self.model.startswith("eu.anthropic"):
+            return Anthropic
+
+        return Default
+                        
+    async def handle(self, msg):
 
         v = msg.value()
 
@@ -101,126 +267,23 @@ class Processor(ConsumerProducer):
 
         print(f"Handling prompt {id}...", flush=True)
 
-        prompt = v.system + "\n\n" + v.prompt
-
         try:
 
-            # Mistral Input Format
+            promptbody = self.variant.encode_request(v.system, v.prompt)
-            if self.model.startswith("mistral"):
-                promptbody = json.dumps({
-                    "prompt": prompt,
-                    "max_tokens": self.max_output,
-                    "temperature": self.temperature,
-                    "top_p": 0.99,
-                    "top_k": 40
-                })
-
-            # Llama 3.1 Input Format
-            elif self.model.startswith("meta"):
-                promptbody = json.dumps({
-                    "prompt": prompt,
-                    "max_gen_len": self.max_output,
-                    "temperature": self.temperature,
-                    "top_p": 0.95,
-                })
-
-            # Anthropic Input Format
-            elif self.model.startswith("anthropic"):
-                promptbody = json.dumps({
-                    "anthropic_version": "bedrock-2023-05-31",
-                    "max_tokens": self.max_output,
-                    "temperature": self.temperature,
-                    "top_p": 0.999,
-                    "messages": [
-                        {
-                            "role": "user",
-                            "content": [
-                                {
-                                    "type": "text",
-                                    "text": prompt
-                                }
-                            ]
-                        }
-                    ]
-                })
-
-            # Jamba Input Format
-            elif self.model.startswith("ai21"):
-                promptbody = json.dumps({
-                    "max_tokens": self.max_output,
-                    "temperature": self.temperature,
-                    "top_p": 0.9,
-                    "messages": [
-                        {
-                            "role": "user",
-                            "content": prompt
-                        }
-                    ]
-                })
-
-            # Cohere Input Format
-            elif self.model.startswith("cohere"):
-                promptbody = json.dumps({
-                    "max_tokens": self.max_output,
-                    "temperature": self.temperature,
-                    "message": prompt
-                })
-
-            # Use Mistral format as defualt
-            else:
-                promptbody = json.dumps({
-                    "prompt": prompt,
-                    "max_tokens": self.max_output,
-                    "temperature": self.temperature,
-                    "top_p": 0.99,
-                    "top_k": 40
-                })
 
             accept = 'application/json'
             contentType = 'application/json'
 
-            # FIXME: Consider catching request limits and raise TooManyRequests
-            # See https://boto3.amazonaws.com/v1/documentation/api/latest/guide/retries.html
-
             with __class__.text_completion_metric.time():
                 response = self.bedrock.invoke_model(
-                    body=promptbody, modelId=self.model, accept=accept,
+                    body=promptbody,
+                    modelId=self.model,
+                    accept=accept,
                     contentType=contentType
                 )
 
-            # Mistral Response Structure
+            # Response structure decode
-            if self.model.startswith("mistral"):
+            outputtext = self.variant.decode_response(response)
-                response_body = json.loads(response.get("body").read())
-                outputtext = response_body['outputs'][0]['text']
-
-            # Claude Response Structure
-            elif self.model.startswith("anthropic"):
-                model_response = json.loads(response["body"].read())
-                outputtext = model_response['content'][0]['text']
-
-            # Llama 3.1 Response Structure
-            elif self.model.startswith("meta"):
-                model_response = json.loads(response["body"].read())
-                outputtext = model_response["generation"]
-
-            # Jamba Response Structure
-            elif self.model.startswith("ai21"):
-                content = response['body'].read()
-                content_str = content.decode('utf-8')
-                content_json = json.loads(content_str)
-                outputtext = content_json['choices'][0]['message']['content']
-
-            # Cohere Input Format
-            elif self.model.startswith("cohere"):
-                content = response['body'].read()
-                content_str = content.decode('utf-8')
-                content_json = json.loads(content_str)
-                outputtext = content_json['text']
-
-            # Use Mistral as default
-            else:
-                response_body = json.loads(response.get("body").read())
-                outputtext = response_body['outputs'][0]['text']
 
             metadata = response['ResponseMetadata']['HTTPHeaders']
             inputtokens = int(metadata['x-amzn-bedrock-input-token-count'])
@@ -243,30 +306,18 @@ class Processor(ConsumerProducer):
 
             print("Done.", flush=True)
 
+        except self.bedrock.exceptions.ThrottlingException as e:
 
-        # FIXME: Wrong exception, don't know what Bedrock throws
+            print("Hit rate limit:", e, flush=True)
-        # for a rate limit
-        except TooManyRequests:
 
-            print("Send rate limit response...", flush=True)
+            # Leave rate limit retries to the base handler
-
+            raise TooManyRequests()
-            r = TextCompletionResponse(
-                error=Error(
-                    type = "rate-limit",
-                    message = str(e),
-                ),
-                response=None,
-                in_token=None,
-                out_token=None,
-                model=None,
-            )
-
-            self.producer.send(r, properties={"id": id})
-
-            self.consumer.acknowledge(msg)
 
         except Exception as e:
 
+            # Apart from rate limits, treat all exceptions as unrecoverable
+
+            print(type(e))
             print(f"Exception: {e}")
 
             print("Send error response...", flush=True)
@@ -299,21 +350,27 @@ class Processor(ConsumerProducer):
         )
 
         parser.add_argument(
-            '-z', '--aws-id-key',
+            '-z', '--aws-access-key-id',
-            default=default_aws_id_key,
+            default=default_access_key_id,
-            help=f'AWS ID Key'
+            help=f'AWS access key ID'
         )
 
         parser.add_argument(
-            '-k', '--aws-secret',
+            '-k', '--aws-secret-access-key',
-            default=default_aws_secret,
+            default=default_secret_access_key,
-            help=f'AWS Secret Key'
+            help=f'AWS secret access key'
         )
 
         parser.add_argument(
             '-r', '--aws-region',
-            default=default_aws_region,
+            default=default_region,
-            help=f'AWS Region'
+            help=f'AWS region'
+        )
+
+        parser.add_argument(
+            '--aws-profile', '--profile', 
+            default=default_profile,
+            help=f'AWS profile name'
         )
 
         parser.add_argument(
@@ -332,5 +389,5 @@ class Processor(ConsumerProducer):
 
 def run():
 
-    Processor.start(module, __doc__)
+    Processor.launch(module, __doc__)
 

trustgraph-cli/scripts/tg-dump-msgpack

@@ -9,6 +9,7 @@ diagnostic utility.
 import msgpack
 import sys
 import argparse
+import json
 
 def dump(input_file, action):
 
@@ -17,7 +18,7 @@ def dump(input_file, action):
         unpacker = msgpack.Unpacker(f, raw=False)
 
         for unpacked in unpacker:
-            print(unpacked)
+            print(json.dumps(unpacked))
 
 def summary(input_file, action):
 

trustgraph-cli/scripts/tg-graph-show

@@ -6,23 +6,23 @@ Connects to the graph query service and dumps all graph edges.
 
 import argparse
 import os
-from trustgraph.clients.triples_query_client import TriplesQueryClient
+from trustgraph.api import Api
 
-default_pulsar_host = os.getenv("PULSAR_HOST", 'pulsar://localhost:6650')
+default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
 default_user = 'trustgraph'
 default_collection = 'default'
 
-def show_graph(pulsar, user, collection):
+def show_graph(url, user, collection):
 
-    tq = TriplesQueryClient(pulsar_host=pulsar)
+    api = Api(url)
 
-    rows = tq.request(
+    rows = api.triples_query(
-        user=user, collection=collection,
+#         user=user, collection=collection,
-        s=None, p=None, o=None, limit=10_000_000
+        s=None, p=None, o=None, limit=10_000,
     )
 
     for row in rows:
-        print(row.s.value, row.p.value, row.o.value)
+        print(row.s, row.p, row.o)
 
 def main():
 
@@ -32,19 +32,19 @@ def main():
     )
 
     parser.add_argument(
-        '-p', '--pulsar-host',
+        '-u', '--api-url',
-        default=default_pulsar_host,
+        default=default_url,
-        help=f'Pulsar host (default: {default_pulsar_host})',
+        help=f'API URL (default: {default_url})',
     )
 
     parser.add_argument(
-        '-u', '--user',
+        '-U', '--user',
         default=default_user,
         help=f'User ID (default: {default_user})'
     )
 
     parser.add_argument(
-        '-c', '--collection',
+        '-C', '--collection',
         default=default_collection,
         help=f'Collection ID (default: {default_collection})'
     )
@@ -54,7 +54,8 @@ def main():
     try:
 
         show_graph(
-            pulsar=args.pulsar_host, user=args.user,
+            url=args.api_url,
+            user=args.user,
             collection=args.collection,
         )
 

trustgraph-cli/scripts/tg-graph-to-turtle

@@ -5,37 +5,45 @@ Connects to the graph query service and dumps all graph edges in Turtle
 format.
 """
 
-import argparse
-import os
-from trustgraph.clients.triples_query_client import TriplesQueryClient
 import rdflib
 import io
 import sys
+import argparse
+import os
 
-default_pulsar_host = os.getenv("PULSAR_HOST", 'pulsar://localhost:6650')
+from trustgraph.api import Api, Uri
 
-def show_graph(pulsar):
+default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
+default_user = 'trustgraph'
+default_collection = 'default'
 
-    tq = TriplesQueryClient(pulsar_host=pulsar)
+def show_graph(url, user, collection):
 
-    rows = tq.request(None, None, None, limit=10_000_000)
+    api = Api(url)
+
+    rows = api.triples_query(
+        s=None, p=None, o=None,
+        limit=10_000)
+#         user=user, collection=collection,
 
     g = rdflib.Graph()
 
     for row in rows:
 
-        sv = rdflib.term.URIRef(row.s.value)
+        sv = rdflib.term.URIRef(row.s)
-        pv = rdflib.term.URIRef(row.p.value)
+        pv = rdflib.term.URIRef(row.p)
 
-        if row.o.is_uri:
+        if isinstance(row.o, Uri):
 
             # Skip malformed URLs with spaces in
-            if " " in row.o.value:
+            if " " in row.o:
                 continue
 
-            ov = rdflib.term.URIRef(row.o.value)
+            ov = rdflib.term.URIRef(row.o)
+
         else:
-            ov = rdflib.term.Literal(row.o.value)
+
+            ov = rdflib.term.Literal(row.o)
 
         g.add((sv, pv, ov))
 
@@ -56,16 +64,32 @@ def main():
     )
 
     parser.add_argument(
-        '-p', '--pulsar-host',
+        '-u', '--api-url',
-        default=default_pulsar_host,
+        default=default_url,
-        help=f'Pulsar host (default: {default_pulsar_host})',
+        help=f'API URL (default: {default_url})',
+    )
+
+    parser.add_argument(
+        '-U', '--user',
+        default=default_user,
+        help=f'User ID (default: {default_user})'
+    )
+
+    parser.add_argument(
+        '-C', '--collection',
+        default=default_collection,
+        help=f'Collection ID (default: {default_collection})'
     )
 
     args = parser.parse_args()
 
     try:
 
-        show_graph(args.pulsar_host)
+        show_graph(
+            url=args.api_url,
+            user=args.user,
+            collection=args.collection
+        )
 
     except Exception as e: