mirror of
https://github.com/trustgraph-ai/trustgraph.git
synced 2026-07-22 11:41:02 +02:00
Merge branch 'release/v0.21' into pulsar-api-support
This commit is contained in:
Tyler O
commit
5bdb9c1919
149 changed files with 3916 additions and 1823 deletions
95
.github/workflows/release.yaml
vendored
95
.github/workflows/release.yaml
vendored
|
|
@ -5,17 +5,17 @@ on:
|
|||
workflow_dispatch:
|
||||
push:
|
||||
tags:
|
||||
- v0.18.*
|
||||
- v*
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
|
||||
deploy:
|
||||
python-packages:
|
||||
|
||||
name: Build everything
|
||||
runs-on: ubuntu-latest
|
||||
name: Release Python packages
|
||||
runs-on: ubuntu-24.04
|
||||
permissions:
|
||||
contents: write
|
||||
id-token: write
|
||||
|
|
@ -25,50 +25,73 @@ jobs:
|
|||
steps:
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: Log in to Docker Hub
|
||||
uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a
|
||||
with:
|
||||
username: ${{ vars.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_SECRET }}
|
||||
|
||||
- name: Install build dependencies
|
||||
run: pip3 install jsonnet
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Get version
|
||||
id: version
|
||||
run: echo VERSION=$(git describe --exact-match --tags | sed 's/^v//') >> $GITHUB_OUTPUT
|
||||
|
||||
- run: echo ${{ steps.version.outputs.VERSION }}
|
||||
|
||||
- name: Build packages
|
||||
run: make packages VERSION=${{ steps.version.outputs.VERSION }}
|
||||
|
||||
- name: Publish release distributions to PyPI
|
||||
uses: pypa/gh-action-pypi-publish@release/v1
|
||||
|
||||
- name: Extract metadata for container
|
||||
id: meta
|
||||
uses: docker/metadata-action@v4
|
||||
with:
|
||||
images: trustgraph/trustgraph-flow
|
||||
tags: |
|
||||
type=ref,event=branch
|
||||
type=ref,event=pr
|
||||
type=semver,pattern={{version}}
|
||||
type=semver,pattern={{major}}.{{minor}}
|
||||
type=sha
|
||||
deploy-container-image:
|
||||
|
||||
- name: Build and push Docker image
|
||||
id: push
|
||||
uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671
|
||||
with:
|
||||
context: .
|
||||
file: ./Containerfile
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
name: Release container image
|
||||
runs-on: ubuntu-24.04
|
||||
permissions:
|
||||
contents: write
|
||||
id-token: write
|
||||
environment:
|
||||
name: release
|
||||
|
||||
steps:
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Docker Hub token
|
||||
run: echo ${{ secrets.DOCKER_SECRET }} > docker-token.txt
|
||||
|
||||
- name: Authenticate with Docker hub
|
||||
run: make docker-hub-login
|
||||
|
||||
- name: Get version
|
||||
id: version
|
||||
run: echo VERSION=$(git describe --exact-match --tags | sed 's/^v//') >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Put version into package manifests
|
||||
run: make update-package-versions VERSION=${{ steps.version.outputs.VERSION }}
|
||||
|
||||
- name: Build containers
|
||||
run: make container VERSION=${{ steps.version.outputs.VERSION }}
|
||||
|
||||
- name: Push containers
|
||||
run: make push VERSION=${{ steps.version.outputs.VERSION }}
|
||||
|
||||
release-bundle:
|
||||
|
||||
name: Upload release bundle
|
||||
runs-on: ubuntu-24.04
|
||||
permissions:
|
||||
contents: write
|
||||
id-token: write
|
||||
environment:
|
||||
name: release
|
||||
|
||||
steps:
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Install build dependencies
|
||||
run: pip3 install jsonnet
|
||||
|
||||
- name: Get version
|
||||
id: version
|
||||
run: echo VERSION=$(git describe --exact-match --tags | sed 's/^v//') >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Create deploy bundle
|
||||
run: templates/generate-all deploy.zip ${{ steps.version.outputs.VERSION }}
|
||||
|
|
|
|||
|
|
@ -11,15 +11,26 @@ ENV PIP_BREAK_SYSTEM_PACKAGES=1
|
|||
RUN dnf install -y python3 python3-pip python3-wheel python3-aiohttp \
|
||||
python3-rdflib
|
||||
|
||||
RUN pip3 install torch --index-url https://download.pytorch.org/whl/cpu
|
||||
RUN pip3 install torch==2.5.1+cpu \
|
||||
--index-url https://download.pytorch.org/whl/cpu
|
||||
|
||||
RUN pip3 install anthropic boto3 cohere openai google-cloud-aiplatform ollama google-generativeai \
|
||||
langchain langchain-core langchain-huggingface langchain-text-splitters \
|
||||
langchain-community pymilvus sentence-transformers transformers \
|
||||
huggingface-hub pulsar-client cassandra-driver pyyaml \
|
||||
RUN pip3 install \
|
||||
anthropic boto3 cohere openai google-cloud-aiplatform \
|
||||
ollama google-generativeai \
|
||||
langchain==0.3.13 langchain-core==0.3.28 langchain-huggingface==0.1.2 \
|
||||
langchain-text-splitters==0.3.4 \
|
||||
langchain-community==0.3.13 \
|
||||
sentence-transformers==3.4.0 transformers==4.47.1 \
|
||||
huggingface-hub==0.27.0 \
|
||||
pymilvus \
|
||||
pulsar-client==3.5.0 cassandra-driver pyyaml \
|
||||
neo4j tiktoken falkordb && \
|
||||
pip3 cache purge
|
||||
|
||||
# Most commonly used embeddings model, just build it into the container
|
||||
# image
|
||||
RUN huggingface-cli download sentence-transformers/all-MiniLM-L6-v2
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# Build a container which contains the built Python packages. The build
|
||||
# creates a bunch of left-over cruft, a separate phase means this is only
|
||||
|
|
|
|||
22
Makefile
22
Makefile
|
|
@ -30,7 +30,7 @@ packages: update-package-versions
|
|||
pypi-upload:
|
||||
twine upload dist/*-${VERSION}.*
|
||||
|
||||
CONTAINER=docker.io/trustgraph/trustgraph-flow
|
||||
CONTAINER_BASE=docker.io/trustgraph
|
||||
|
||||
update-package-versions:
|
||||
mkdir -p trustgraph-cli/trustgraph
|
||||
|
|
@ -44,11 +44,23 @@ update-package-versions:
|
|||
echo __version__ = \"${VERSION}\" > trustgraph/trustgraph/trustgraph_version.py
|
||||
|
||||
container: update-package-versions
|
||||
${DOCKER} build -f Containerfile -t ${CONTAINER}:${VERSION} \
|
||||
--format docker
|
||||
${DOCKER} build -f containers/Containerfile.base \
|
||||
-t ${CONTAINER_BASE}/trustgraph-base:${VERSION} .
|
||||
${DOCKER} build -f containers/Containerfile.flow \
|
||||
-t ${CONTAINER_BASE}/trustgraph-flow:${VERSION} .
|
||||
${DOCKER} build -f containers/Containerfile.bedrock \
|
||||
-t ${CONTAINER_BASE}/trustgraph-bedrock:${VERSION} .
|
||||
${DOCKER} build -f containers/Containerfile.vertexai \
|
||||
-t ${CONTAINER_BASE}/trustgraph-vertexai:${VERSION} .
|
||||
${DOCKER} build -f containers/Containerfile.hf \
|
||||
-t ${CONTAINER_BASE}/trustgraph-hf:${VERSION} .
|
||||
|
||||
push:
|
||||
${DOCKER} push ${CONTAINER}:${VERSION}
|
||||
${DOCKER} push ${CONTAINER_BASE}/trustgraph-base:${VERSION}
|
||||
${DOCKER} push ${CONTAINER_BASE}/trustgraph-flow:${VERSION}
|
||||
${DOCKER} push ${CONTAINER_BASE}/trustgraph-bedrock:${VERSION}
|
||||
${DOCKER} push ${CONTAINER_BASE}/trustgraph-vertexai:${VERSION}
|
||||
${DOCKER} push ${CONTAINER_BASE}/trustgraph-hf:${VERSION}
|
||||
|
||||
clean:
|
||||
rm -rf wheels/
|
||||
|
|
@ -104,5 +116,5 @@ update-dcs: set-version
|
|||
|
||||
docker-hub-login:
|
||||
cat docker-token.txt | \
|
||||
docker login -u trustgraph --password-stdin registry-1.docker.io
|
||||
${DOCKER} login -u trustgraph --password-stdin registry-1.docker.io
|
||||
|
||||
|
|
|
|||
48
containers/Containerfile.base
Normal file
48
containers/Containerfile.base
Normal file
|
|
@ -0,0 +1,48 @@
|
|||
|
||||
# ----------------------------------------------------------------------------
|
||||
# Build an AI container. This does the torch install which is huge, and I
|
||||
# like to avoid re-doing this.
|
||||
# ----------------------------------------------------------------------------
|
||||
|
||||
FROM docker.io/fedora:40 AS base
|
||||
|
||||
ENV PIP_BREAK_SYSTEM_PACKAGES=1
|
||||
|
||||
RUN dnf install -y python3 python3-pip python3-wheel python3-aiohttp && \
|
||||
dnf clean all
|
||||
|
||||
RUN pip3 install --no-cache-dir pulsar-client==3.5.0
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# Build a container which contains the built Python packages. The build
|
||||
# creates a bunch of left-over cruft, a separate phase means this is only
|
||||
# needed to support package build
|
||||
# ----------------------------------------------------------------------------
|
||||
|
||||
FROM base AS build
|
||||
|
||||
COPY trustgraph-base/ /root/build/trustgraph-base/
|
||||
COPY trustgraph-cli/ /root/build/trustgraph-cli/
|
||||
|
||||
WORKDIR /root/build/
|
||||
|
||||
RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-base/
|
||||
RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-cli/
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# Finally, the target container. Start with base and add the package.
|
||||
# ----------------------------------------------------------------------------
|
||||
|
||||
FROM base
|
||||
|
||||
COPY --from=build /root/wheels /root/wheels
|
||||
|
||||
RUN \
|
||||
pip3 install --no-cache-dir /root/wheels/trustgraph_base-* && \
|
||||
pip3 install --no-cache-dir /root/wheels/trustgraph_cli-* && \
|
||||
rm -rf /root/wheels
|
||||
|
||||
WORKDIR /
|
||||
|
||||
|
||||
|
||||
48
containers/Containerfile.bedrock
Normal file
48
containers/Containerfile.bedrock
Normal file
|
|
@ -0,0 +1,48 @@
|
|||
|
||||
# ----------------------------------------------------------------------------
|
||||
# Build an AI container. This does the torch install which is huge, and I
|
||||
# like to avoid re-doing this.
|
||||
# ----------------------------------------------------------------------------
|
||||
|
||||
FROM docker.io/fedora:40 AS base
|
||||
|
||||
ENV PIP_BREAK_SYSTEM_PACKAGES=1
|
||||
|
||||
RUN dnf install -y python3 python3-pip python3-wheel python3-aiohttp \
|
||||
python3-rdflib
|
||||
|
||||
RUN pip3 install --no-cache-dir boto3 pulsar-client==3.5.0
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# Build a container which contains the built Python packages. The build
|
||||
# creates a bunch of left-over cruft, a separate phase means this is only
|
||||
# needed to support package build
|
||||
# ----------------------------------------------------------------------------
|
||||
|
||||
FROM base AS build
|
||||
|
||||
COPY trustgraph-base/ /root/build/trustgraph-base/
|
||||
COPY trustgraph-bedrock/ /root/build/trustgraph-bedrock/
|
||||
|
||||
WORKDIR /root/build/
|
||||
|
||||
RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-base/
|
||||
RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-bedrock/
|
||||
|
||||
RUN ls /root/wheels
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# Finally, the target container. Start with base and add the package.
|
||||
# ----------------------------------------------------------------------------
|
||||
|
||||
FROM base
|
||||
|
||||
COPY --from=build /root/wheels /root/wheels
|
||||
|
||||
RUN \
|
||||
pip3 install --no-cache-dir /root/wheels/trustgraph_base-* && \
|
||||
pip3 install --no-cache-dir /root/wheels/trustgraph_bedrock-* && \
|
||||
rm -rf /root/wheels
|
||||
|
||||
WORKDIR /
|
||||
|
||||
60
containers/Containerfile.flow
Normal file
60
containers/Containerfile.flow
Normal file
|
|
@ -0,0 +1,60 @@
|
|||
|
||||
# ----------------------------------------------------------------------------
|
||||
# Build an AI container. This does the torch install which is huge, and I
|
||||
# like to avoid re-doing this.
|
||||
# ----------------------------------------------------------------------------
|
||||
|
||||
FROM docker.io/fedora:40 AS base
|
||||
|
||||
ENV PIP_BREAK_SYSTEM_PACKAGES=1
|
||||
|
||||
RUN dnf install -y python3 python3-pip python3-wheel python3-aiohttp \
|
||||
python3-rdflib
|
||||
|
||||
RUN pip3 install --no-cache-dir \
|
||||
anthropic cohere openai google-generativeai \
|
||||
ollama \
|
||||
langchain==0.3.13 langchain-core==0.3.28 \
|
||||
langchain-text-splitters==0.3.4 \
|
||||
langchain-community==0.3.13 \
|
||||
pymilvus \
|
||||
pulsar-client==3.5.0 cassandra-driver pyyaml \
|
||||
neo4j tiktoken falkordb && \
|
||||
pip3 cache purge
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# Build a container which contains the built Python packages. The build
|
||||
# creates a bunch of left-over cruft, a separate phase means this is only
|
||||
# needed to support package build
|
||||
# ----------------------------------------------------------------------------
|
||||
|
||||
FROM base AS build
|
||||
|
||||
COPY trustgraph-base/ /root/build/trustgraph-base/
|
||||
COPY trustgraph-flow/ /root/build/trustgraph-flow/
|
||||
COPY trustgraph-cli/ /root/build/trustgraph-cli/
|
||||
|
||||
WORKDIR /root/build/
|
||||
|
||||
RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-base/
|
||||
RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-flow/
|
||||
RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-cli/
|
||||
|
||||
RUN ls /root/wheels
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# Finally, the target container. Start with base and add the package.
|
||||
# ----------------------------------------------------------------------------
|
||||
|
||||
FROM base
|
||||
|
||||
COPY --from=build /root/wheels /root/wheels
|
||||
|
||||
RUN \
|
||||
pip3 install --no-cache-dir /root/wheels/trustgraph_base-* && \
|
||||
pip3 install --no-cache-dir /root/wheels/trustgraph_flow-* && \
|
||||
pip3 install --no-cache-dir /root/wheels/trustgraph_cli-* && \
|
||||
rm -rf /root/wheels
|
||||
|
||||
WORKDIR /
|
||||
|
||||
75
containers/Containerfile.hf
Normal file
75
containers/Containerfile.hf
Normal file
|
|
@ -0,0 +1,75 @@
|
|||
|
||||
# ----------------------------------------------------------------------------
|
||||
# Build an AI container. This does the torch install which is huge, and I
|
||||
# like to avoid re-doing this.
|
||||
# ----------------------------------------------------------------------------
|
||||
|
||||
FROM docker.io/fedora:40 AS ai
|
||||
|
||||
ENV PIP_BREAK_SYSTEM_PACKAGES=1
|
||||
|
||||
RUN dnf install -y python3 python3-pip python3-wheel python3-aiohttp \
|
||||
python3-rdflib
|
||||
|
||||
RUN pip3 install torch==2.5.1+cpu \
|
||||
--index-url https://download.pytorch.org/whl/cpu
|
||||
|
||||
RUN pip3 install --no-cache-dir \
|
||||
langchain==0.3.13 langchain-core==0.3.28 langchain-huggingface==0.1.2 \
|
||||
langchain-community==0.3.13 \
|
||||
sentence-transformers==3.4.0 transformers==4.47.1 \
|
||||
huggingface-hub==0.27.0 \
|
||||
pulsar-client==3.5.0
|
||||
|
||||
# Most commonly used embeddings model, just build it into the container
|
||||
# image
|
||||
RUN huggingface-cli download sentence-transformers/all-MiniLM-L6-v2
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# Build a container which contains the built Python packages. The build
|
||||
# creates a bunch of left-over cruft, a separate phase means this is only
|
||||
# needed to support package build
|
||||
# ----------------------------------------------------------------------------
|
||||
|
||||
FROM ai AS build
|
||||
|
||||
COPY trustgraph-base/ /root/build/trustgraph-base/
|
||||
COPY trustgraph-flow/ /root/build/trustgraph-flow/
|
||||
COPY trustgraph-vertexai/ /root/build/trustgraph-vertexai/
|
||||
COPY trustgraph-bedrock/ /root/build/trustgraph-bedrock/
|
||||
COPY trustgraph-embeddings-hf/ /root/build/trustgraph-embeddings-hf/
|
||||
COPY trustgraph-cli/ /root/build/trustgraph-cli/
|
||||
|
||||
WORKDIR /root/build/
|
||||
|
||||
RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-base/
|
||||
RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-flow/
|
||||
RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-vertexai/
|
||||
RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-bedrock/
|
||||
RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-embeddings-hf/
|
||||
RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-cli/
|
||||
|
||||
RUN ls /root/wheels
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# Finally, the target container. Start with base and add the package.
|
||||
# ----------------------------------------------------------------------------
|
||||
|
||||
FROM ai
|
||||
|
||||
COPY --from=build /root/wheels /root/wheels
|
||||
|
||||
RUN \
|
||||
pip3 install /root/wheels/trustgraph_base-* && \
|
||||
pip3 install /root/wheels/trustgraph_flow-* && \
|
||||
pip3 install /root/wheels/trustgraph_vertexai-* && \
|
||||
pip3 install /root/wheels/trustgraph_bedrock-* && \
|
||||
pip3 install /root/wheels/trustgraph_embeddings_hf-* && \
|
||||
pip3 install /root/wheels/trustgraph_cli-* && \
|
||||
pip3 cache purge && \
|
||||
rm -rf /root/wheels
|
||||
|
||||
WORKDIR /
|
||||
|
||||
CMD sleep 1000000
|
||||
|
||||
51
containers/Containerfile.vertexai
Normal file
51
containers/Containerfile.vertexai
Normal file
|
|
@ -0,0 +1,51 @@
|
|||
|
||||
# ----------------------------------------------------------------------------
|
||||
# Build an AI container. This does the torch install which is huge, and I
|
||||
# like to avoid re-doing this.
|
||||
# ----------------------------------------------------------------------------
|
||||
|
||||
FROM docker.io/fedora:40 AS base
|
||||
|
||||
ENV PIP_BREAK_SYSTEM_PACKAGES=1
|
||||
|
||||
RUN dnf install -y python3 python3-pip python3-wheel python3-aiohttp \
|
||||
python3-rdflib
|
||||
|
||||
RUN pip3 install --no-cache-dir \
|
||||
google-cloud-aiplatform pulsar-client==3.5.0
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# Build a container which contains the built Python packages. The build
|
||||
# creates a bunch of left-over cruft, a separate phase means this is only
|
||||
# needed to support package build
|
||||
# ----------------------------------------------------------------------------
|
||||
|
||||
FROM base AS build
|
||||
|
||||
COPY trustgraph-base/ /root/build/trustgraph-base/
|
||||
COPY trustgraph-vertexai/ /root/build/trustgraph-vertexai/
|
||||
|
||||
WORKDIR /root/build/
|
||||
|
||||
RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-base/
|
||||
RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-vertexai/
|
||||
|
||||
RUN ls /root/wheels
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# Finally, the target container. Start with base and add the package.
|
||||
# ----------------------------------------------------------------------------
|
||||
|
||||
FROM base
|
||||
|
||||
COPY --from=build /root/wheels /root/wheels
|
||||
|
||||
RUN \
|
||||
pip3 install --no-cache-dir /root/wheels/trustgraph_base-* && \
|
||||
pip3 install --no-cache-dir /root/wheels/trustgraph_vertexai-* && \
|
||||
rm -rf /root/wheels
|
||||
|
||||
WORKDIR /
|
||||
|
||||
|
||||
|
||||
|
|
@ -113,8 +113,9 @@ Choose one of the `Docker Compose` files that meets your preferred model and gra
|
|||
### AWS Bedrock API
|
||||
|
||||
```
|
||||
export AWS_ID_KEY=<ID-KEY-HERE>
|
||||
export AWS_SECRET_KEY=<TOKEN-GOES-HERE>
|
||||
export AWS_ACCESS_KEY_ID=<ID-KEY-HERE>
|
||||
export AWS_SECRET_ACCESS_KEY=<TOKEN-GOES-HERE>
|
||||
export AWS_DEFAULT_REGION=<REGION-HERE>
|
||||
docker compose -f tg-launch-bedrock-cassandra.yaml up -d # Using Cassandra as the graph store
|
||||
docker compose -f tg-launch-bedrock-neo4j.yaml up -d # Using Neo4j as the graph store
|
||||
```
|
||||
|
|
|
|||
|
|
@ -577,7 +577,7 @@
|
|||
"disableTextWrap": false,
|
||||
"editorMode": "builder",
|
||||
"exemplar": false,
|
||||
"expr": "increase(processing_count_total{status!=\"success\"}[$__rate_interval])",
|
||||
"expr": "sum by(job) (increase(rate_limit_count_total[$__rate_interval]))",
|
||||
"format": "time_series",
|
||||
"fullMetaSearch": false,
|
||||
"includeNullMetadata": true,
|
||||
|
|
@ -588,7 +588,7 @@
|
|||
"useBackend": false
|
||||
}
|
||||
],
|
||||
"title": "Errors",
|
||||
"title": "Rate limit events",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
|
|
|
|||
|
|
@ -20,6 +20,18 @@ scrape_configs:
|
|||
- targets:
|
||||
- 'pulsar:8080'
|
||||
|
||||
- job_name: 'bookie'
|
||||
scrape_interval: 5s
|
||||
static_configs:
|
||||
- targets:
|
||||
- 'bookie:8000'
|
||||
|
||||
- job_name: 'zookeeper'
|
||||
scrape_interval: 5s
|
||||
static_configs:
|
||||
- targets:
|
||||
- 'zookeeper:8000'
|
||||
|
||||
- job_name: 'pdf-decoder'
|
||||
scrape_interval: 5s
|
||||
static_configs:
|
||||
|
|
@ -32,11 +44,17 @@ scrape_configs:
|
|||
- targets:
|
||||
- 'chunker:8000'
|
||||
|
||||
- job_name: 'vectorize'
|
||||
- job_name: 'document-embeddings'
|
||||
scrape_interval: 5s
|
||||
static_configs:
|
||||
- targets:
|
||||
- 'vectorize:8000'
|
||||
- 'document-embeddings:8000'
|
||||
|
||||
- job_name: 'graph-embeddings'
|
||||
scrape_interval: 5s
|
||||
static_configs:
|
||||
- targets:
|
||||
- 'graph-embeddings:8000'
|
||||
|
||||
- job_name: 'embeddings'
|
||||
scrape_interval: 5s
|
||||
|
|
@ -74,6 +92,12 @@ scrape_configs:
|
|||
- targets:
|
||||
- 'metering-rag:8000'
|
||||
|
||||
- job_name: 'store-doc-embeddings'
|
||||
scrape_interval: 5s
|
||||
static_configs:
|
||||
- targets:
|
||||
- 'store-doc-embeddings:8000'
|
||||
|
||||
- job_name: 'store-graph-embeddings'
|
||||
scrape_interval: 5s
|
||||
static_configs:
|
||||
|
|
@ -104,6 +128,12 @@ scrape_configs:
|
|||
- targets:
|
||||
- 'graph-rag:8000'
|
||||
|
||||
- job_name: 'document-rag'
|
||||
scrape_interval: 5s
|
||||
static_configs:
|
||||
- targets:
|
||||
- 'document-rag:8000'
|
||||
|
||||
- job_name: 'prompt'
|
||||
scrape_interval: 5s
|
||||
static_configs:
|
||||
|
|
@ -122,6 +152,12 @@ scrape_configs:
|
|||
- targets:
|
||||
- 'query-graph-embeddings:8000'
|
||||
|
||||
- job_name: 'query-doc-embeddings'
|
||||
scrape_interval: 5s
|
||||
static_configs:
|
||||
- targets:
|
||||
- 'query-doc-embeddings:8000'
|
||||
|
||||
- job_name: 'query-triples'
|
||||
scrape_interval: 5s
|
||||
static_configs:
|
||||
|
|
@ -145,3 +181,7 @@ scrape_configs:
|
|||
static_configs:
|
||||
- targets:
|
||||
- 'workbench-ui:8000'
|
||||
|
||||
# Cassandra
|
||||
# qdrant
|
||||
|
||||
|
|
|
|||
|
|
@ -1,47 +1,70 @@
|
|||
{
|
||||
|
||||
// Essentials
|
||||
"trustgraph-base": import "components/trustgraph.jsonnet",
|
||||
"pulsar": import "components/pulsar.jsonnet",
|
||||
|
||||
// LLMs
|
||||
"azure": import "components/azure.jsonnet",
|
||||
"azure-openai": import "components/azure-openai.jsonnet",
|
||||
"bedrock": import "components/bedrock.jsonnet",
|
||||
"claude": import "components/claude.jsonnet",
|
||||
"cohere": import "components/cohere.jsonnet",
|
||||
"document-rag": import "components/document-rag.jsonnet",
|
||||
"embeddings-hf": import "components/embeddings-hf.jsonnet",
|
||||
"embeddings-ollama": import "components/embeddings-ollama.jsonnet",
|
||||
"googleaistudio": import "components/googleaistudio.jsonnet",
|
||||
"grafana": import "components/grafana.jsonnet",
|
||||
"ollama": import "components/ollama.jsonnet",
|
||||
"openai": import "components/openai.jsonnet",
|
||||
"vertexai": import "components/vertexai.jsonnet",
|
||||
|
||||
// LLMs for RAG
|
||||
"azure-rag": import "components/azure-rag.jsonnet",
|
||||
"azure-openai-rag": import "components/azure-openai-rag.jsonnet",
|
||||
"bedrock-rag": import "components/bedrock-rag.jsonnet",
|
||||
"claude-rag": import "components/claude-rag.jsonnet",
|
||||
"cohere-rag": import "components/cohere-rag.jsonnet",
|
||||
"googleaistudio-rag": import "components/googleaistudio-rag.jsonnet",
|
||||
"ollama-rag": import "components/ollama-rag.jsonnet",
|
||||
"openai-rag": import "components/openai-rag.jsonnet",
|
||||
"vertexai-rag": import "components/vertexai-rag.jsonnet",
|
||||
|
||||
// Embeddings
|
||||
"embeddings-ollama": import "components/embeddings-ollama.jsonnet",
|
||||
"embeddings-hf": import "components/embeddings-hf.jsonnet",
|
||||
|
||||
// Processing pipelines
|
||||
"graph-rag": import "components/graph-rag.jsonnet",
|
||||
"document-rag": import "components/document-rag.jsonnet",
|
||||
|
||||
// Vector stores
|
||||
"vector-store-milvus": import "components/milvus.jsonnet",
|
||||
"vector-store-qdrant": import "components/qdrant.jsonnet",
|
||||
"vector-store-pinecone": import "components/pinecone.jsonnet",
|
||||
|
||||
// Triples stores
|
||||
"triple-store-cassandra": import "components/cassandra.jsonnet",
|
||||
"triple-store-neo4j": import "components/neo4j.jsonnet",
|
||||
"triple-store-falkordb": import "components/falkordb.jsonnet",
|
||||
"triple-store-memgraph": import "components/memgraph.jsonnet",
|
||||
|
||||
// Observability support
|
||||
"grafana": import "components/grafana.jsonnet",
|
||||
|
||||
// Pulsar manager is a UI for Pulsar. Uses a LOT of memory
|
||||
"pulsar-manager": import "components/pulsar-manager.jsonnet",
|
||||
|
||||
"llamafile": import "components/llamafile.jsonnet",
|
||||
"ollama": import "components/ollama.jsonnet",
|
||||
"openai": import "components/openai.jsonnet",
|
||||
"override-recursive-chunker": import "components/chunker-recursive.jsonnet",
|
||||
|
||||
// The prompt manager
|
||||
"prompt-template": import "components/prompt-template.jsonnet",
|
||||
"prompt-overrides": import "components/prompt-overrides.jsonnet",
|
||||
|
||||
"pulsar": import "components/pulsar.jsonnet",
|
||||
"pulsar-manager": import "components/pulsar-manager.jsonnet",
|
||||
"trustgraph-base": import "components/trustgraph.jsonnet",
|
||||
"vector-store-milvus": import "components/milvus.jsonnet",
|
||||
"vector-store-qdrant": import "components/qdrant.jsonnet",
|
||||
"vector-store-pinecone": import "components/pinecone.jsonnet",
|
||||
"vertexai": import "components/vertexai.jsonnet",
|
||||
"workbench-ui": import "components/workbench-ui.jsonnet",
|
||||
"null": {},
|
||||
|
||||
// ReAct agent
|
||||
"agent-manager-react": import "components/agent-manager-react.jsonnet",
|
||||
|
||||
// FIXME: Dupes
|
||||
"cassandra": import "components/cassandra.jsonnet",
|
||||
"neo4j": import "components/neo4j.jsonnet",
|
||||
"memgraph": import "components/memgraph.jsonnet",
|
||||
"qdrant": import "components/qdrant.jsonnet",
|
||||
"pinecone": import "components/pinecone.jsonnet",
|
||||
"milvus": import "components/milvus.jsonnet",
|
||||
"falkordb": import "components/falkordb.jsonnet",
|
||||
"trustgraph": import "components/trustgraph.jsonnet",
|
||||
// Optional UI
|
||||
"workbench-ui": import "components/workbench-ui.jsonnet",
|
||||
|
||||
// Does nothing. But, can be a hack to overwrite parameters
|
||||
"null": {},
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,11 +14,15 @@ local default_prompts = import "prompts/default-prompts.jsonnet";
|
|||
|
||||
local container =
|
||||
engine.container("agent-manager")
|
||||
.with_image(images.trustgraph)
|
||||
.with_image(images.trustgraph_flow)
|
||||
.with_command([
|
||||
"agent-manager-react",
|
||||
"-p",
|
||||
url.pulsar,
|
||||
"--prompt-request-queue",
|
||||
"non-persistent://tg/request/prompt-rag",
|
||||
"--prompt-response-queue",
|
||||
"non-persistent://tg/response/prompt-rag",
|
||||
"--tool-type",
|
||||
] + [
|
||||
tool.id + "=" + tool.type
|
||||
|
|
|
|||
61
templates/components/azure-openai-rag.jsonnet
Normal file
61
templates/components/azure-openai-rag.jsonnet
Normal file
|
|
@ -0,0 +1,61 @@
|
|||
local base = import "base/base.jsonnet";
|
||||
local images = import "values/images.jsonnet";
|
||||
local url = import "values/url.jsonnet";
|
||||
local prompts = import "prompts/mixtral.jsonnet";
|
||||
|
||||
{
|
||||
|
||||
with:: function(key, value)
|
||||
self + {
|
||||
["ollama-rag-" + key]:: value,
|
||||
},
|
||||
|
||||
"azure-openai-rag-model":: "GPT-3.5-Turbo",
|
||||
"azure-openai-rag-max-output-tokens":: 4192,
|
||||
"azure-openai-rag-temperature":: 0.0,
|
||||
|
||||
"text-completion-rag" +: {
|
||||
|
||||
create:: function(engine)
|
||||
|
||||
local envSecrets = engine.envSecrets("azure-openai-credentials")
|
||||
.with_env_var("AZURE_TOKEN", "azure-token");
|
||||
|
||||
local containerRag =
|
||||
engine.container("text-completion-rag")
|
||||
.with_image(images.trustgraph_flow)
|
||||
.with_command([
|
||||
"text-completion-azure",
|
||||
"-p",
|
||||
url.pulsar,
|
||||
"-x",
|
||||
std.toString($["azure-openai-rag-max-output-tokens"]),
|
||||
"-t",
|
||||
"%0.3f" % $["azure-openai-rag-temperature"],
|
||||
"-i",
|
||||
"non-persistent://tg/request/text-completion-rag",
|
||||
"-o",
|
||||
"non-persistent://tg/response/text-completion-rag",
|
||||
])
|
||||
.with_env_var_secrets(envSecrets)
|
||||
.with_limits("0.5", "128M")
|
||||
.with_reservations("0.1", "128M");
|
||||
|
||||
local containerSetRag = engine.containers(
|
||||
"text-completion-rag", [ containerRag ]
|
||||
);
|
||||
|
||||
local serviceRag =
|
||||
engine.internalService(containerSetRag)
|
||||
.with_port(8000, 8000, "metrics");
|
||||
|
||||
engine.resources([
|
||||
envSecrets,
|
||||
containerSetRag,
|
||||
serviceRag,
|
||||
])
|
||||
|
||||
},
|
||||
|
||||
} + prompts
|
||||
|
||||
|
|
@ -5,6 +5,11 @@ local prompts = import "prompts/mixtral.jsonnet";
|
|||
|
||||
{
|
||||
|
||||
with:: function(key, value)
|
||||
self + {
|
||||
["azure-openai-" + key]:: value,
|
||||
},
|
||||
|
||||
"azure-openai-model":: "GPT-3.5-Turbo",
|
||||
"azure-openai-max-output-tokens":: 4192,
|
||||
"azure-openai-temperature":: 0.0,
|
||||
|
|
@ -18,7 +23,7 @@ local prompts = import "prompts/mixtral.jsonnet";
|
|||
|
||||
local container =
|
||||
engine.container("text-completion")
|
||||
.with_image(images.trustgraph)
|
||||
.with_image(images.trustgraph_flow)
|
||||
.with_command([
|
||||
"text-completion-azure-openai",
|
||||
"-p",
|
||||
|
|
@ -34,48 +39,18 @@ local prompts = import "prompts/mixtral.jsonnet";
|
|||
.with_limits("0.5", "128M")
|
||||
.with_reservations("0.1", "128M");
|
||||
|
||||
local containerRag =
|
||||
engine.container("text-completion-rag")
|
||||
.with_image(images.trustgraph)
|
||||
.with_command([
|
||||
"text-completion-azure",
|
||||
"-p",
|
||||
url.pulsar,
|
||||
"-x",
|
||||
std.toString($["azure-openai-max-output-tokens"]),
|
||||
"-t",
|
||||
"%0.3f" % $["azure-openai-temperature"],
|
||||
"-i",
|
||||
"non-persistent://tg/request/text-completion-rag",
|
||||
"-o",
|
||||
"non-persistent://tg/response/text-completion-rag",
|
||||
])
|
||||
.with_env_var_secrets(envSecrets)
|
||||
.with_limits("0.5", "128M")
|
||||
.with_reservations("0.1", "128M");
|
||||
|
||||
local containerSet = engine.containers(
|
||||
"text-completion", [ container ]
|
||||
);
|
||||
|
||||
local containerSetRag = engine.containers(
|
||||
"text-completion-rag", [ containerRag ]
|
||||
);
|
||||
|
||||
local service =
|
||||
engine.internalService(containerSet)
|
||||
.with_port(8000, 8000, "metrics");
|
||||
|
||||
local serviceRag =
|
||||
engine.internalService(containerSetRag)
|
||||
.with_port(8000, 8000, "metrics");
|
||||
|
||||
engine.resources([
|
||||
envSecrets,
|
||||
containerSet,
|
||||
containerSetRag,
|
||||
service,
|
||||
serviceRag,
|
||||
])
|
||||
|
||||
},
|
||||
|
|
|
|||
60
templates/components/azure-rag.jsonnet
Normal file
60
templates/components/azure-rag.jsonnet
Normal file
|
|
@ -0,0 +1,60 @@
|
|||
local images = import "values/images.jsonnet";
|
||||
local url = import "values/url.jsonnet";
|
||||
local prompts = import "prompts/mixtral.jsonnet";
|
||||
|
||||
{
|
||||
|
||||
with:: function(key, value)
|
||||
self + {
|
||||
["azure-rag-" + key]:: value,
|
||||
},
|
||||
|
||||
"azure-rag-max-output-tokens":: 4096,
|
||||
"azure-rag-temperature":: 0.0,
|
||||
|
||||
"text-completion-rag" +: {
|
||||
|
||||
create:: function(engine)
|
||||
|
||||
local envSecrets = engine.envSecrets("azure-credentials")
|
||||
.with_env_var("AZURE_TOKEN", "azure-token")
|
||||
.with_env_var("AZURE_ENDPOINT", "azure-endpoint");
|
||||
|
||||
local containerRag =
|
||||
engine.container("text-completion-rag")
|
||||
.with_image(images.trustgraph_flow)
|
||||
.with_command([
|
||||
"text-completion-azure",
|
||||
"-p",
|
||||
url.pulsar,
|
||||
"-x",
|
||||
std.toString($["azure-rag-max-output-tokens"]),
|
||||
"-t",
|
||||
"%0.3f" % $["azure-rag-temperature"],
|
||||
"-i",
|
||||
"non-persistent://tg/request/text-completion-rag",
|
||||
"-o",
|
||||
"non-persistent://tg/response/text-completion-rag",
|
||||
])
|
||||
.with_env_var_secrets(envSecrets)
|
||||
.with_limits("0.5", "128M")
|
||||
.with_reservations("0.1", "128M");
|
||||
|
||||
local containerSetRag = engine.containers(
|
||||
"text-completion-rag", [ containerRag ]
|
||||
);
|
||||
|
||||
local serviceRag =
|
||||
engine.internalService(containerSetRag)
|
||||
.with_port(8000, 8000, "metrics");
|
||||
|
||||
engine.resources([
|
||||
envSecrets,
|
||||
containerSetRag,
|
||||
serviceRag,
|
||||
])
|
||||
|
||||
}
|
||||
|
||||
} + prompts
|
||||
|
||||
|
|
@ -1,10 +1,14 @@
|
|||
local base = import "base/base.jsonnet";
|
||||
local images = import "values/images.jsonnet";
|
||||
local url = import "values/url.jsonnet";
|
||||
local prompts = import "prompts/mixtral.jsonnet";
|
||||
|
||||
{
|
||||
|
||||
with:: function(key, value)
|
||||
self + {
|
||||
["azure-" + key]:: value,
|
||||
},
|
||||
|
||||
"azure-max-output-tokens":: 4096,
|
||||
"azure-temperature":: 0.0,
|
||||
|
||||
|
|
@ -18,7 +22,7 @@ local prompts = import "prompts/mixtral.jsonnet";
|
|||
|
||||
local container =
|
||||
engine.container("text-completion")
|
||||
.with_image(images.trustgraph)
|
||||
.with_image(images.trustgraph_flow)
|
||||
.with_command([
|
||||
"text-completion-azure",
|
||||
"-p",
|
||||
|
|
@ -32,48 +36,18 @@ local prompts = import "prompts/mixtral.jsonnet";
|
|||
.with_limits("0.5", "128M")
|
||||
.with_reservations("0.1", "128M");
|
||||
|
||||
local containerRag =
|
||||
engine.container("text-completion-rag")
|
||||
.with_image(images.trustgraph)
|
||||
.with_command([
|
||||
"text-completion-azure",
|
||||
"-p",
|
||||
url.pulsar,
|
||||
"-x",
|
||||
std.toString($["azure-max-output-tokens"]),
|
||||
"-t",
|
||||
"%0.3f" % $["azure-temperature"],
|
||||
"-i",
|
||||
"non-persistent://tg/request/text-completion-rag",
|
||||
"-o",
|
||||
"non-persistent://tg/response/text-completion-rag",
|
||||
])
|
||||
.with_env_var_secrets(envSecrets)
|
||||
.with_limits("0.5", "128M")
|
||||
.with_reservations("0.1", "128M");
|
||||
|
||||
local containerSet = engine.containers(
|
||||
"text-completion", [ container ]
|
||||
);
|
||||
|
||||
local containerSetRag = engine.containers(
|
||||
"text-completion-rag", [ containerRag ]
|
||||
);
|
||||
|
||||
local service =
|
||||
engine.internalService(containerSet)
|
||||
.with_port(8000, 8000, "metrics");
|
||||
|
||||
local serviceRag =
|
||||
engine.internalService(containerSetRag)
|
||||
.with_port(8000, 8000, "metrics");
|
||||
|
||||
engine.resources([
|
||||
envSecrets,
|
||||
containerSet,
|
||||
containerSetRag,
|
||||
service,
|
||||
serviceRag,
|
||||
])
|
||||
|
||||
}
|
||||
|
|
|
|||
66
templates/components/bedrock-rag.jsonnet
Normal file
66
templates/components/bedrock-rag.jsonnet
Normal file
|
|
@ -0,0 +1,66 @@
|
|||
local base = import "base/base.jsonnet";
|
||||
local images = import "values/images.jsonnet";
|
||||
local url = import "values/url.jsonnet";
|
||||
local prompts = import "prompts/mixtral.jsonnet";
|
||||
local chunker = import "chunker-recursive.jsonnet";
|
||||
|
||||
{
|
||||
|
||||
with:: function(key, value)
|
||||
self + {
|
||||
["bedrock-rag-" + key]:: value,
|
||||
},
|
||||
|
||||
"bedrock-rag-max-output-tokens":: 4096,
|
||||
"bedrock-rag-temperature":: 0.0,
|
||||
"bedrock-rag-model":: "mistral.mixtral-8x7b-instruct-v0:1",
|
||||
|
||||
"text-completion-rag" +: {
|
||||
|
||||
create:: function(engine)
|
||||
|
||||
local envSecrets = engine.envSecrets("bedrock-credentials")
|
||||
.with_env_var("AWS_ACCESS_KEY_ID", "aws-id-key")
|
||||
.with_env_var("AWS_SECRET_ACCESS_KEY", "aws-secret")
|
||||
.with_env_var("AWS_DEFAULT_REGION", "aws-region");
|
||||
|
||||
local containerRag =
|
||||
engine.container("text-completion-rag")
|
||||
.with_image(images.trustgraph_bedrock)
|
||||
.with_command([
|
||||
"text-completion-bedrock",
|
||||
"-p",
|
||||
url.pulsar,
|
||||
"-x",
|
||||
std.toString($["bedrock-rag-max-output-tokens"]),
|
||||
"-t",
|
||||
"%0.3f" % $["bedrock-rag-temperature"],
|
||||
"-m",
|
||||
$["bedrock-rag-model"],
|
||||
"-i",
|
||||
"non-persistent://tg/request/text-completion-rag",
|
||||
"-o",
|
||||
"non-persistent://tg/response/text-completion-rag",
|
||||
])
|
||||
.with_env_var_secrets(envSecrets)
|
||||
.with_limits("0.5", "128M")
|
||||
.with_reservations("0.1", "128M");
|
||||
|
||||
local containerSetRag = engine.containers(
|
||||
"text-completion-rag", [ containerRag ]
|
||||
);
|
||||
|
||||
local serviceRag =
|
||||
engine.internalService(containerSetRag)
|
||||
.with_port(8000, 8000, "metrics");
|
||||
|
||||
engine.resources([
|
||||
envSecrets,
|
||||
containerSetRag,
|
||||
serviceRag,
|
||||
])
|
||||
|
||||
},
|
||||
|
||||
} + prompts + chunker
|
||||
|
||||
|
|
@ -6,6 +6,11 @@ local chunker = import "chunker-recursive.jsonnet";
|
|||
|
||||
{
|
||||
|
||||
with:: function(key, value)
|
||||
self + {
|
||||
["bedrock-" + key]:: value,
|
||||
},
|
||||
|
||||
"bedrock-max-output-tokens":: 4096,
|
||||
"bedrock-temperature":: 0.0,
|
||||
"bedrock-model":: "mistral.mixtral-8x7b-instruct-v0:1",
|
||||
|
|
@ -15,13 +20,13 @@ local chunker = import "chunker-recursive.jsonnet";
|
|||
create:: function(engine)
|
||||
|
||||
local envSecrets = engine.envSecrets("bedrock-credentials")
|
||||
.with_env_var("AWS_ID_KEY", "aws-id-key")
|
||||
.with_env_var("AWS_SECRET", "aws-secret")
|
||||
.with_env_var("AWS_REGION", "aws-region");
|
||||
.with_env_var("AWS_ACCESS_KEY_ID", "aws-id-key")
|
||||
.with_env_var("AWS_SECRET_ACCESS_KEY", "aws-secret")
|
||||
.with_env_var("AWS_DEFAULT_REGION", "aws-region");
|
||||
|
||||
local container =
|
||||
engine.container("text-completion")
|
||||
.with_image(images.trustgraph)
|
||||
.with_image(images.trustgraph_bedrock)
|
||||
.with_command([
|
||||
"text-completion-bedrock",
|
||||
"-p",
|
||||
|
|
@ -37,50 +42,18 @@ local chunker = import "chunker-recursive.jsonnet";
|
|||
.with_limits("0.5", "128M")
|
||||
.with_reservations("0.1", "128M");
|
||||
|
||||
local containerRag =
|
||||
engine.container("text-completion-rag")
|
||||
.with_image(images.trustgraph)
|
||||
.with_command([
|
||||
"text-completion-bedrock",
|
||||
"-p",
|
||||
url.pulsar,
|
||||
"-x",
|
||||
std.toString($["bedrock-max-output-tokens"]),
|
||||
"-t",
|
||||
"%0.3f" % $["bedrock-temperature"],
|
||||
"-m",
|
||||
$["bedrock-model"],
|
||||
"-i",
|
||||
"non-persistent://tg/request/text-completion-rag",
|
||||
"-o",
|
||||
"non-persistent://tg/response/text-completion-rag",
|
||||
])
|
||||
.with_env_var_secrets(envSecrets)
|
||||
.with_limits("0.5", "128M")
|
||||
.with_reservations("0.1", "128M");
|
||||
|
||||
local containerSet = engine.containers(
|
||||
"text-completion", [ container ]
|
||||
);
|
||||
|
||||
local containerSetRag = engine.containers(
|
||||
"text-completion-rag", [ containerRag ]
|
||||
);
|
||||
|
||||
local service =
|
||||
engine.internalService(containerSet)
|
||||
.with_port(8000, 8000, "metrics");
|
||||
|
||||
local serviceRag =
|
||||
engine.internalService(containerSetRag)
|
||||
.with_port(8000, 8000, "metrics");
|
||||
|
||||
engine.resources([
|
||||
envSecrets,
|
||||
containerSet,
|
||||
containerSetRag,
|
||||
service,
|
||||
serviceRag,
|
||||
])
|
||||
|
||||
},
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ cassandra + {
|
|||
|
||||
local container =
|
||||
engine.container("store-triples")
|
||||
.with_image(images.trustgraph)
|
||||
.with_image(images.trustgraph_flow)
|
||||
.with_command([
|
||||
"triples-write-cassandra",
|
||||
"-p",
|
||||
|
|
@ -44,7 +44,7 @@ cassandra + {
|
|||
|
||||
local container =
|
||||
engine.container("query-triples")
|
||||
.with_image(images.trustgraph)
|
||||
.with_image(images.trustgraph_flow)
|
||||
.with_command([
|
||||
"triples-query-cassandra",
|
||||
"-p",
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ local prompts = import "prompts/mixtral.jsonnet";
|
|||
|
||||
local container =
|
||||
engine.container("chunker")
|
||||
.with_image(images.trustgraph)
|
||||
.with_image(images.trustgraph_flow)
|
||||
.with_command([
|
||||
"chunker-recursive",
|
||||
"-p",
|
||||
|
|
|
|||
63
templates/components/claude-rag.jsonnet
Normal file
63
templates/components/claude-rag.jsonnet
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
local base = import "base/base.jsonnet";
|
||||
local images = import "values/images.jsonnet";
|
||||
local url = import "values/url.jsonnet";
|
||||
local prompts = import "prompts/mixtral.jsonnet";
|
||||
|
||||
{
|
||||
|
||||
with:: function(key, value)
|
||||
self + {
|
||||
["claude-rag-" + key]:: value,
|
||||
},
|
||||
|
||||
"claude-rag-model":: "claude-3-sonnet-20240229",
|
||||
"claude-rag-max-output-tokens":: 4096,
|
||||
"claude-rag-temperature":: 0.0,
|
||||
|
||||
"text-completion-rag" +: {
|
||||
|
||||
create:: function(engine)
|
||||
|
||||
local envSecrets = engine.envSecrets("claude-credentials")
|
||||
.with_env_var("CLAUDE_KEY", "claude-key");
|
||||
|
||||
local containerRag =
|
||||
engine.container("text-completion-rag")
|
||||
.with_image(images.trustgraph_flow)
|
||||
.with_command([
|
||||
"text-completion-claude",
|
||||
"-p",
|
||||
url.pulsar,
|
||||
"-x",
|
||||
std.toString($["claude-rag-max-output-tokens"]),
|
||||
"-m",
|
||||
$["claude-rag-model"],
|
||||
"-t",
|
||||
"%0.3f" % $["claude-rag-temperature"],
|
||||
"-i",
|
||||
"non-persistent://tg/request/text-completion-rag",
|
||||
"-o",
|
||||
"non-persistent://tg/response/text-completion-rag",
|
||||
])
|
||||
.with_env_var_secrets(envSecrets)
|
||||
.with_limits("0.5", "128M")
|
||||
.with_reservations("0.1", "128M");
|
||||
|
||||
local containerSetRag = engine.containers(
|
||||
"text-completion-rag", [ containerRag ]
|
||||
);
|
||||
|
||||
local serviceRag =
|
||||
engine.internalService(containerSetRag)
|
||||
.with_port(8000, 8000, "metrics");
|
||||
|
||||
engine.resources([
|
||||
envSecrets,
|
||||
containerSetRag,
|
||||
serviceRag,
|
||||
])
|
||||
|
||||
},
|
||||
|
||||
} + prompts
|
||||
|
||||
|
|
@ -5,6 +5,12 @@ local prompts = import "prompts/mixtral.jsonnet";
|
|||
|
||||
{
|
||||
|
||||
with:: function(key, value)
|
||||
self + {
|
||||
["claude-" + key]:: value,
|
||||
},
|
||||
|
||||
"claude-model":: "claude-3-sonnet-20240229",
|
||||
"claude-max-output-tokens":: 4096,
|
||||
"claude-temperature":: 0.0,
|
||||
|
||||
|
|
@ -13,17 +19,19 @@ local prompts = import "prompts/mixtral.jsonnet";
|
|||
create:: function(engine)
|
||||
|
||||
local envSecrets = engine.envSecrets("claude-credentials")
|
||||
.with_env_var("CLAUDE_KEY_TOKEN", "claude-key");
|
||||
.with_env_var("CLAUDE_KEY", "claude-key");
|
||||
|
||||
local container =
|
||||
engine.container("text-completion")
|
||||
.with_image(images.trustgraph)
|
||||
.with_image(images.trustgraph_flow)
|
||||
.with_command([
|
||||
"text-completion-claude",
|
||||
"-p",
|
||||
url.pulsar,
|
||||
"-x",
|
||||
std.toString($["claude-max-output-tokens"]),
|
||||
"-m",
|
||||
$["claude-model"],
|
||||
"-t",
|
||||
"%0.3f" % $["claude-temperature"],
|
||||
])
|
||||
|
|
@ -31,48 +39,18 @@ local prompts = import "prompts/mixtral.jsonnet";
|
|||
.with_limits("0.5", "128M")
|
||||
.with_reservations("0.1", "128M");
|
||||
|
||||
local containerRag =
|
||||
engine.container("text-completion-rag")
|
||||
.with_image(images.trustgraph)
|
||||
.with_command([
|
||||
"text-completion-claude",
|
||||
"-p",
|
||||
url.pulsar,
|
||||
"-x",
|
||||
std.toString($["claude-max-output-tokens"]),
|
||||
"-t",
|
||||
"%0.3f" % $["claude-temperature"],
|
||||
"-i",
|
||||
"non-persistent://tg/request/text-completion-rag",
|
||||
"-o",
|
||||
"non-persistent://tg/response/text-completion-rag",
|
||||
])
|
||||
.with_env_var_secrets(envSecrets)
|
||||
.with_limits("0.5", "128M")
|
||||
.with_reservations("0.1", "128M");
|
||||
|
||||
local containerSet = engine.containers(
|
||||
"text-completion", [ container ]
|
||||
);
|
||||
|
||||
local containerSetRag = engine.containers(
|
||||
"text-completion-rag", [ containerRag ]
|
||||
);
|
||||
|
||||
local service =
|
||||
engine.internalService(containerSet)
|
||||
.with_port(8000, 8000, "metrics");
|
||||
|
||||
local serviceRag =
|
||||
engine.internalService(containerSetRag)
|
||||
.with_port(8000, 8000, "metrics");
|
||||
|
||||
engine.resources([
|
||||
envSecrets,
|
||||
containerSet,
|
||||
containerSetRag,
|
||||
service,
|
||||
serviceRag,
|
||||
])
|
||||
|
||||
},
|
||||
|
|
|
|||
56
templates/components/cohere-rag.jsonnet
Normal file
56
templates/components/cohere-rag.jsonnet
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
local base = import "base/base.jsonnet";
|
||||
local images = import "values/images.jsonnet";
|
||||
local url = import "values/url.jsonnet";
|
||||
local prompts = import "prompts/mixtral.jsonnet";
|
||||
|
||||
{
|
||||
|
||||
with:: function(key, value)
|
||||
self + {
|
||||
["cohere-rag-" + key]:: value,
|
||||
},
|
||||
|
||||
"cohere-rag-temperature":: 0.0,
|
||||
|
||||
"text-completion-rag" +: {
|
||||
|
||||
create:: function(engine)
|
||||
|
||||
local envSecrets = engine.envSecrets("cohere-credentials")
|
||||
.with_env_var("COHERE_KEY", "cohere-key");
|
||||
|
||||
local containerRag =
|
||||
engine.container("text-completion-rag")
|
||||
.with_image(images.trustgraph_flow)
|
||||
.with_command([
|
||||
"text-completion-cohere",
|
||||
"-p",
|
||||
url.pulsar,
|
||||
"-t",
|
||||
"%0.3f" % $["cohere-rag-temperature"],
|
||||
"-i",
|
||||
"non-persistent://tg/request/text-completion-rag",
|
||||
"-o",
|
||||
"non-persistent://tg/response/text-completion-rag",
|
||||
])
|
||||
.with_limits("0.5", "128M")
|
||||
.with_reservations("0.1", "128M");
|
||||
|
||||
local containerSetRag = engine.containers(
|
||||
"text-completion-rag", [ containerRag ]
|
||||
);
|
||||
|
||||
local serviceRag =
|
||||
engine.internalService(containerSetRag)
|
||||
.with_port(8000, 8000, "metrics");
|
||||
|
||||
engine.resources([
|
||||
envSecrets,
|
||||
containerSetRag,
|
||||
serviceRag,
|
||||
])
|
||||
|
||||
},
|
||||
|
||||
} + prompts
|
||||
|
||||
|
|
@ -5,9 +5,10 @@ local prompts = import "prompts/mixtral.jsonnet";
|
|||
|
||||
{
|
||||
|
||||
// Override chunking
|
||||
"chunk-size":: 150,
|
||||
"chunk-overlap":: 10,
|
||||
with:: function(key, value)
|
||||
self + {
|
||||
["cohere-" + key]:: value,
|
||||
},
|
||||
|
||||
"cohere-temperature":: 0.0,
|
||||
|
||||
|
|
@ -20,7 +21,7 @@ local prompts = import "prompts/mixtral.jsonnet";
|
|||
|
||||
local container =
|
||||
engine.container("text-completion")
|
||||
.with_image(images.trustgraph)
|
||||
.with_image(images.trustgraph_flow)
|
||||
.with_command([
|
||||
"text-completion-cohere",
|
||||
"-p",
|
||||
|
|
@ -31,45 +32,18 @@ local prompts = import "prompts/mixtral.jsonnet";
|
|||
.with_limits("0.5", "128M")
|
||||
.with_reservations("0.1", "128M");
|
||||
|
||||
local containerRag =
|
||||
engine.container("text-completion-rag")
|
||||
.with_image(images.trustgraph)
|
||||
.with_command([
|
||||
"text-completion-cohere",
|
||||
"-p",
|
||||
url.pulsar,
|
||||
"-t",
|
||||
"%0.3f" % $["cohere-temperature"],
|
||||
"-i",
|
||||
"non-persistent://tg/request/text-completion-rag",
|
||||
"-o",
|
||||
"non-persistent://tg/response/text-completion-rag",
|
||||
])
|
||||
.with_limits("0.5", "128M")
|
||||
.with_reservations("0.1", "128M");
|
||||
|
||||
local containerSet = engine.containers(
|
||||
"text-completion", [ container ]
|
||||
);
|
||||
|
||||
local containerSetRag = engine.containers(
|
||||
"text-completion-rag", [ containerRag ]
|
||||
);
|
||||
|
||||
local service =
|
||||
engine.internalService(containerSet)
|
||||
.with_port(8000, 8000, "metrics");
|
||||
|
||||
local serviceRag =
|
||||
engine.internalService(containerSetRag)
|
||||
.with_port(8000, 8000, "metrics");
|
||||
|
||||
engine.resources([
|
||||
envSecrets,
|
||||
containerSet,
|
||||
containerSetRag,
|
||||
service,
|
||||
serviceRag,
|
||||
])
|
||||
|
||||
},
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ local prompts = import "prompts/mixtral.jsonnet";
|
|||
|
||||
local container =
|
||||
engine.container("document-rag")
|
||||
.with_image(images.trustgraph)
|
||||
.with_image(images.trustgraph_flow)
|
||||
.with_command([
|
||||
"document-rag",
|
||||
"-p",
|
||||
|
|
@ -39,5 +39,35 @@ local prompts = import "prompts/mixtral.jsonnet";
|
|||
|
||||
},
|
||||
|
||||
"document-embeddings" +: {
|
||||
|
||||
create:: function(engine)
|
||||
|
||||
local container =
|
||||
engine.container("document-embeddings")
|
||||
.with_image(images.trustgraph_flow)
|
||||
.with_command([
|
||||
"document-embeddings",
|
||||
"-p",
|
||||
url.pulsar,
|
||||
])
|
||||
.with_limits("1.0", "512M")
|
||||
.with_reservations("0.5", "512M");
|
||||
|
||||
local containerSet = engine.containers(
|
||||
"document-embeddings", [ container ]
|
||||
);
|
||||
|
||||
local service =
|
||||
engine.internalService(containerSet)
|
||||
.with_port(8000, 8000, "metrics");
|
||||
|
||||
engine.resources([
|
||||
containerSet,
|
||||
service,
|
||||
])
|
||||
|
||||
},
|
||||
|
||||
}
|
||||
|
||||
|
|
|
|||
43
templates/components/embeddings-fastembed.jsonnet
Normal file
43
templates/components/embeddings-fastembed.jsonnet
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
local base = import "base/base.jsonnet";
|
||||
local images = import "values/images.jsonnet";
|
||||
local url = import "values/url.jsonnet";
|
||||
local prompts = import "prompts/mixtral.jsonnet";
|
||||
|
||||
{
|
||||
|
||||
"embeddings-model":: "sentence-transformers/all-MiniLM-L6-v2",
|
||||
|
||||
embeddings +: {
|
||||
|
||||
create:: function(engine)
|
||||
|
||||
local container =
|
||||
engine.container("embeddings")
|
||||
.with_image(images.trustgraph_flow)
|
||||
.with_command([
|
||||
"embeddings-fastembed",
|
||||
"-p",
|
||||
url.pulsar,
|
||||
"-m",
|
||||
$["embeddings-model"],
|
||||
])
|
||||
.with_limits("1.0", "400M")
|
||||
.with_reservations("0.5", "400M");
|
||||
|
||||
local containerSet = engine.containers(
|
||||
"embeddings", [ container ]
|
||||
);
|
||||
|
||||
local service =
|
||||
engine.internalService(containerSet)
|
||||
.with_port(8000, 8000, "metrics");
|
||||
|
||||
engine.resources([
|
||||
containerSet,
|
||||
service,
|
||||
])
|
||||
|
||||
},
|
||||
|
||||
}
|
||||
|
||||
|
|
@ -13,7 +13,7 @@ local prompts = import "prompts/mixtral.jsonnet";
|
|||
|
||||
local container =
|
||||
engine.container("embeddings")
|
||||
.with_image(images.trustgraph)
|
||||
.with_image(images.trustgraph_hf)
|
||||
.with_command([
|
||||
"embeddings-hf",
|
||||
"-p",
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ local url = import "values/url.jsonnet";
|
|||
|
||||
local container =
|
||||
engine.container("embeddings")
|
||||
.with_image(images.trustgraph)
|
||||
.with_image(images.trustgraph_flow)
|
||||
.with_command([
|
||||
"embeddings-ollama",
|
||||
"-p",
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ falkordb + {
|
|||
|
||||
local container =
|
||||
engine.container("store-triples")
|
||||
.with_image(images.trustgraph)
|
||||
.with_image(images.trustgraph_flow)
|
||||
.with_command([
|
||||
"triples-write-falkordb",
|
||||
"-p",
|
||||
|
|
@ -45,7 +45,7 @@ falkordb + {
|
|||
|
||||
local container =
|
||||
engine.container("query-triples")
|
||||
.with_image(images.trustgraph)
|
||||
.with_image(images.trustgraph_flow)
|
||||
.with_command([
|
||||
"triples-query-falkordb",
|
||||
"-p",
|
||||
|
|
|
|||
65
templates/components/googleaistudio-rag.jsonnet
Normal file
65
templates/components/googleaistudio-rag.jsonnet
Normal file
|
|
@ -0,0 +1,65 @@
|
|||
local base = import "base/base.jsonnet";
|
||||
local images = import "values/images.jsonnet";
|
||||
local url = import "values/url.jsonnet";
|
||||
local prompts = import "prompts/mixtral.jsonnet";
|
||||
|
||||
{
|
||||
|
||||
with:: function(key, value)
|
||||
self + {
|
||||
["googleaistudio-rag-" + key]:: value,
|
||||
},
|
||||
|
||||
"googleaistudio-rag-max-output-tokens":: 4096,
|
||||
"googleaistudio-rag-temperature":: 0.0,
|
||||
"googleaistudio-rag-model":: "gemini-1.5-flash-002",
|
||||
|
||||
"text-completion-rag" +: {
|
||||
|
||||
create:: function(engine)
|
||||
|
||||
local envSecrets = engine.envSecrets("googleaistudio-credentials")
|
||||
.with_env_var("GOOGLE_AI_STUDIO_KEY", "googleaistudio-key");
|
||||
|
||||
local containerRag =
|
||||
engine.container("text-completion-rag")
|
||||
.with_image(images.trustgraph_flow)
|
||||
.with_command([
|
||||
"text-completion-googleaistudio",
|
||||
"-p",
|
||||
url.pulsar,
|
||||
"-x",
|
||||
std.toString(
|
||||
$["googleaistudio-rag-max-output-tokens"]
|
||||
),
|
||||
"-t",
|
||||
"%0.3f" % $["googleaistudio-rag-temperature"],
|
||||
"-m",
|
||||
$["googleaistudio-rag-model"],
|
||||
"-i",
|
||||
"non-persistent://tg/request/text-completion-rag",
|
||||
"-o",
|
||||
"non-persistent://tg/response/text-completion-rag",
|
||||
])
|
||||
.with_env_var_secrets(envSecrets)
|
||||
.with_limits("0.5", "128M")
|
||||
.with_reservations("0.1", "128M");
|
||||
|
||||
local containerSetRag = engine.containers(
|
||||
"text-completion-rag", [ containerRag ]
|
||||
);
|
||||
|
||||
local serviceRag =
|
||||
engine.internalService(containerSetRag)
|
||||
.with_port(8000, 8000, "metrics");
|
||||
|
||||
engine.resources([
|
||||
envSecrets,
|
||||
containerSetRag,
|
||||
serviceRag,
|
||||
])
|
||||
|
||||
},
|
||||
|
||||
} + prompts
|
||||
|
||||
|
|
@ -5,6 +5,11 @@ local prompts = import "prompts/mixtral.jsonnet";
|
|||
|
||||
{
|
||||
|
||||
with:: function(key, value)
|
||||
self + {
|
||||
["googleaistudio-" + key]:: value,
|
||||
},
|
||||
|
||||
"googleaistudio-max-output-tokens":: 4096,
|
||||
"googleaistudio-temperature":: 0.0,
|
||||
"googleaistudio-model":: "gemini-1.5-flash-002",
|
||||
|
|
@ -13,12 +18,12 @@ local prompts = import "prompts/mixtral.jsonnet";
|
|||
|
||||
create:: function(engine)
|
||||
|
||||
local envSecrets = engine.envSecrets("googleaistudio-key")
|
||||
local envSecrets = engine.envSecrets("googleaistudio-credentials")
|
||||
.with_env_var("GOOGLE_AI_STUDIO_KEY", "googleaistudio-key");
|
||||
|
||||
local container =
|
||||
engine.container("text-completion")
|
||||
.with_image(images.trustgraph)
|
||||
.with_image(images.trustgraph_flow)
|
||||
.with_command([
|
||||
"text-completion-googleaistudio",
|
||||
"-p",
|
||||
|
|
@ -34,50 +39,18 @@ local prompts = import "prompts/mixtral.jsonnet";
|
|||
.with_limits("0.5", "128M")
|
||||
.with_reservations("0.1", "128M");
|
||||
|
||||
local containerRag =
|
||||
engine.container("text-completion-rag")
|
||||
.with_image(images.trustgraph)
|
||||
.with_command([
|
||||
"text-completion-googleaistudio",
|
||||
"-p",
|
||||
url.pulsar,
|
||||
"-x",
|
||||
std.toString($["googleaistudio-max-output-tokens"]),
|
||||
"-t",
|
||||
"%0.3f" % $["googleaistudio-temperature"],
|
||||
"-m",
|
||||
$["googleaistudio-model"],
|
||||
"-i",
|
||||
"non-persistent://tg/request/text-completion-rag",
|
||||
"-o",
|
||||
"non-persistent://tg/response/text-completion-rag",
|
||||
])
|
||||
.with_env_var_secrets(envSecrets)
|
||||
.with_limits("0.5", "128M")
|
||||
.with_reservations("0.1", "128M");
|
||||
|
||||
local containerSet = engine.containers(
|
||||
"text-completion", [ container ]
|
||||
);
|
||||
|
||||
local containerSetRag = engine.containers(
|
||||
"text-completion-rag", [ containerRag ]
|
||||
);
|
||||
|
||||
local service =
|
||||
engine.internalService(containerSet)
|
||||
.with_port(8000, 8000, "metrics");
|
||||
|
||||
local serviceRag =
|
||||
engine.internalService(containerSetRag)
|
||||
.with_port(8000, 8000, "metrics");
|
||||
|
||||
engine.resources([
|
||||
envSecrets,
|
||||
containerSet,
|
||||
containerSetRag,
|
||||
service,
|
||||
serviceRag,
|
||||
])
|
||||
|
||||
},
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ local url = import "values/url.jsonnet";
|
|||
|
||||
local container =
|
||||
engine.container("kg-extract-definitions")
|
||||
.with_image(images.trustgraph)
|
||||
.with_image(images.trustgraph_flow)
|
||||
.with_command([
|
||||
"kg-extract-definitions",
|
||||
"-p",
|
||||
|
|
@ -44,7 +44,7 @@ local url = import "values/url.jsonnet";
|
|||
|
||||
local container =
|
||||
engine.container("kg-extract-relationships")
|
||||
.with_image(images.trustgraph)
|
||||
.with_image(images.trustgraph_flow)
|
||||
.with_command([
|
||||
"kg-extract-relationships",
|
||||
"-p",
|
||||
|
|
@ -74,7 +74,7 @@ local url = import "values/url.jsonnet";
|
|||
|
||||
local container =
|
||||
engine.container("kg-extract-topics")
|
||||
.with_image(images.trustgraph)
|
||||
.with_image(images.trustgraph_flow)
|
||||
.with_command([
|
||||
"kg-extract-topics",
|
||||
"-p",
|
||||
|
|
@ -104,7 +104,7 @@ local url = import "values/url.jsonnet";
|
|||
|
||||
local container =
|
||||
engine.container("graph-rag")
|
||||
.with_image(images.trustgraph)
|
||||
.with_image(images.trustgraph_flow)
|
||||
.with_command([
|
||||
"graph-rag",
|
||||
"-p",
|
||||
|
|
@ -138,5 +138,35 @@ local url = import "values/url.jsonnet";
|
|||
|
||||
},
|
||||
|
||||
"graph-embeddings" +: {
|
||||
|
||||
create:: function(engine)
|
||||
|
||||
local container =
|
||||
engine.container("graph-embeddings")
|
||||
.with_image(images.trustgraph_flow)
|
||||
.with_command([
|
||||
"graph-embeddings",
|
||||
"-p",
|
||||
url.pulsar,
|
||||
])
|
||||
.with_limits("1.0", "512M")
|
||||
.with_reservations("0.5", "512M");
|
||||
|
||||
local containerSet = engine.containers(
|
||||
"graph-embeddings", [ container ]
|
||||
);
|
||||
|
||||
local service =
|
||||
engine.internalService(containerSet)
|
||||
.with_port(8000, 8000, "metrics");
|
||||
|
||||
engine.resources([
|
||||
containerSet,
|
||||
service,
|
||||
])
|
||||
|
||||
},
|
||||
|
||||
}
|
||||
|
||||
|
|
|
|||
57
templates/components/llamafile-rag.jsonnet
Normal file
57
templates/components/llamafile-rag.jsonnet
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
local base = import "base/base.jsonnet";
|
||||
local images = import "values/images.jsonnet";
|
||||
local url = import "values/url.jsonnet";
|
||||
local prompts = import "prompts/slm.jsonnet";
|
||||
|
||||
{
|
||||
|
||||
with:: function(key, value)
|
||||
self + {
|
||||
["llamafile-rag-" + key]:: value,
|
||||
},
|
||||
|
||||
"llamafile-rag-model":: "LLaMA_CPP",
|
||||
|
||||
"text-completion-rag" +: {
|
||||
|
||||
create:: function(engine)
|
||||
|
||||
local envSecrets = engine.envSecrets("llamafile-credentials")
|
||||
.with_env_var("LLAMAFILE_URL", "llamafile-url");
|
||||
|
||||
local containerRag =
|
||||
engine.container("text-completion-rag")
|
||||
.with_image(images.trustgraph_flow)
|
||||
.with_command([
|
||||
"text-completion-llamafile",
|
||||
"-p",
|
||||
url.pulsar,
|
||||
"-m",
|
||||
$["llamafile-rag-model"],
|
||||
"-i",
|
||||
"non-persistent://tg/request/text-completion-rag",
|
||||
"-o",
|
||||
"non-persistent://tg/response/text-completion-rag",
|
||||
])
|
||||
.with_env_var_secrets(envSecrets)
|
||||
.with_limits("0.5", "128M")
|
||||
.with_reservations("0.1", "128M");
|
||||
|
||||
local containerSetRag = engine.containers(
|
||||
"text-completion-rag", [ containerRag ]
|
||||
);
|
||||
|
||||
local serviceRag =
|
||||
engine.internalService(containerSetRag)
|
||||
.with_port(8080, 8080, "metrics");
|
||||
|
||||
engine.resources([
|
||||
envSecrets,
|
||||
containerSetRag,
|
||||
serviceRag,
|
||||
])
|
||||
|
||||
},
|
||||
|
||||
} + prompts
|
||||
|
||||
|
|
@ -5,6 +5,11 @@ local prompts = import "prompts/slm.jsonnet";
|
|||
|
||||
{
|
||||
|
||||
with:: function(key, value)
|
||||
self + {
|
||||
["llamafile-" + key]:: value,
|
||||
},
|
||||
|
||||
"llamafile-model":: "LLaMA_CPP",
|
||||
|
||||
"text-completion" +: {
|
||||
|
|
@ -16,7 +21,7 @@ local prompts = import "prompts/slm.jsonnet";
|
|||
|
||||
local container =
|
||||
engine.container("text-completion")
|
||||
.with_image(images.trustgraph)
|
||||
.with_image(images.trustgraph_flow)
|
||||
.with_command([
|
||||
"text-completion-llamafile",
|
||||
"-p",
|
||||
|
|
@ -28,46 +33,18 @@ local prompts = import "prompts/slm.jsonnet";
|
|||
.with_limits("0.5", "128M")
|
||||
.with_reservations("0.1", "128M");
|
||||
|
||||
local containerRag =
|
||||
engine.container("text-completion-rag")
|
||||
.with_image(images.trustgraph)
|
||||
.with_command([
|
||||
"text-completion-llamafile",
|
||||
"-p",
|
||||
url.pulsar,
|
||||
"-m",
|
||||
$["llamafile-model"],
|
||||
"-i",
|
||||
"non-persistent://tg/request/text-completion-rag",
|
||||
"-o",
|
||||
"non-persistent://tg/response/text-completion-rag",
|
||||
])
|
||||
.with_env_var_secrets(envSecrets)
|
||||
.with_limits("0.5", "128M")
|
||||
.with_reservations("0.1", "128M");
|
||||
|
||||
local containerSet = engine.containers(
|
||||
"text-completion", [ container ]
|
||||
);
|
||||
|
||||
local containerSetRag = engine.containers(
|
||||
"text-completion-rag", [ containerRag ]
|
||||
);
|
||||
|
||||
local service =
|
||||
engine.internalService(containerSet)
|
||||
.with_port(8080, 8080, "metrics");
|
||||
|
||||
local serviceRag =
|
||||
engine.internalService(containerSetRag)
|
||||
.with_port(8080, 8080, "metrics");
|
||||
|
||||
engine.resources([
|
||||
envSecrets,
|
||||
containerSet,
|
||||
containerSetRag,
|
||||
service,
|
||||
serviceRag,
|
||||
])
|
||||
|
||||
},
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ memgraph + {
|
|||
|
||||
local container =
|
||||
engine.container("store-triples")
|
||||
.with_image(images.trustgraph)
|
||||
.with_image(images.trustgraph_flow)
|
||||
.with_command([
|
||||
"triples-write-memgraph",
|
||||
"-p",
|
||||
|
|
@ -48,7 +48,7 @@ memgraph + {
|
|||
|
||||
local container =
|
||||
engine.container("query-triples")
|
||||
.with_image(images.trustgraph)
|
||||
.with_image(images.trustgraph_flow)
|
||||
.with_command([
|
||||
"triples-query-memgraph",
|
||||
"-p",
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ milvus + {
|
|||
|
||||
local container =
|
||||
engine.container("store-graph-embeddings")
|
||||
.with_image(images.trustgraph)
|
||||
.with_image(images.trustgraph_flow)
|
||||
.with_command([
|
||||
"ge-write-milvus",
|
||||
"-p",
|
||||
|
|
@ -44,7 +44,7 @@ milvus + {
|
|||
|
||||
local container =
|
||||
engine.container("query-graph-embeddings")
|
||||
.with_image(images.trustgraph)
|
||||
.with_image(images.trustgraph_flow)
|
||||
.with_command([
|
||||
"ge-query-milvus",
|
||||
"-p",
|
||||
|
|
@ -76,7 +76,7 @@ milvus + {
|
|||
|
||||
local container =
|
||||
engine.container("store-doc-embeddings")
|
||||
.with_image(images.trustgraph)
|
||||
.with_image(images.trustgraph_flow)
|
||||
.with_command([
|
||||
"de-write-milvus",
|
||||
"-p",
|
||||
|
|
@ -108,7 +108,7 @@ milvus + {
|
|||
|
||||
local container =
|
||||
engine.container("query-doc-embeddings")
|
||||
.with_image(images.trustgraph)
|
||||
.with_image(images.trustgraph_flow)
|
||||
.with_command([
|
||||
"de-query-milvus",
|
||||
"-p",
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ neo4j + {
|
|||
|
||||
local container =
|
||||
engine.container("store-triples")
|
||||
.with_image(images.trustgraph)
|
||||
.with_image(images.trustgraph_flow)
|
||||
.with_command([
|
||||
"triples-write-neo4j",
|
||||
"-p",
|
||||
|
|
@ -45,7 +45,7 @@ neo4j + {
|
|||
|
||||
local container =
|
||||
engine.container("query-triples")
|
||||
.with_image(images.trustgraph)
|
||||
.with_image(images.trustgraph_flow)
|
||||
.with_command([
|
||||
"triples-query-neo4j",
|
||||
"-p",
|
||||
|
|
|
|||
57
templates/components/ollama-rag.jsonnet
Normal file
57
templates/components/ollama-rag.jsonnet
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
local base = import "base/base.jsonnet";
|
||||
local images = import "values/images.jsonnet";
|
||||
local url = import "values/url.jsonnet";
|
||||
local prompts = import "prompts/mixtral.jsonnet";
|
||||
|
||||
{
|
||||
|
||||
with:: function(key, value)
|
||||
self + {
|
||||
["ollama-rag-" + key]:: value,
|
||||
},
|
||||
|
||||
"ollama-rag-model":: "gemma2:9b",
|
||||
|
||||
"text-completion-rag" +: {
|
||||
|
||||
create:: function(engine)
|
||||
|
||||
local envSecrets = engine.envSecrets("ollama-credentials")
|
||||
.with_env_var("OLLAMA_HOST", "ollama-host");
|
||||
|
||||
local containerRag =
|
||||
engine.container("text-completion-rag")
|
||||
.with_image(images.trustgraph_flow)
|
||||
.with_command([
|
||||
"text-completion-ollama",
|
||||
"-p",
|
||||
url.pulsar,
|
||||
"-m",
|
||||
$["ollama-rag-model"],
|
||||
"-i",
|
||||
"non-persistent://tg/request/text-completion-rag",
|
||||
"-o",
|
||||
"non-persistent://tg/response/text-completion-rag",
|
||||
])
|
||||
.with_env_var_secrets(envSecrets)
|
||||
.with_limits("0.5", "128M")
|
||||
.with_reservations("0.1", "128M");
|
||||
|
||||
local containerSetRag = engine.containers(
|
||||
"text-completion-rag", [ containerRag ]
|
||||
);
|
||||
|
||||
local serviceRag =
|
||||
engine.internalService(containerSetRag)
|
||||
.with_port(8080, 8080, "metrics");
|
||||
|
||||
engine.resources([
|
||||
envSecrets,
|
||||
containerSetRag,
|
||||
serviceRag,
|
||||
])
|
||||
|
||||
},
|
||||
|
||||
} + prompts
|
||||
|
||||
|
|
@ -5,6 +5,11 @@ local prompts = import "prompts/mixtral.jsonnet";
|
|||
|
||||
{
|
||||
|
||||
with:: function(key, value)
|
||||
self + {
|
||||
["ollama-" + key]:: value,
|
||||
},
|
||||
|
||||
"ollama-model":: "gemma2:9b",
|
||||
|
||||
"text-completion" +: {
|
||||
|
|
@ -16,7 +21,7 @@ local prompts = import "prompts/mixtral.jsonnet";
|
|||
|
||||
local container =
|
||||
engine.container("text-completion")
|
||||
.with_image(images.trustgraph)
|
||||
.with_image(images.trustgraph_flow)
|
||||
.with_command([
|
||||
"text-completion-ollama",
|
||||
"-p",
|
||||
|
|
@ -28,46 +33,18 @@ local prompts = import "prompts/mixtral.jsonnet";
|
|||
.with_limits("0.5", "128M")
|
||||
.with_reservations("0.1", "128M");
|
||||
|
||||
local containerRag =
|
||||
engine.container("text-completion-rag")
|
||||
.with_image(images.trustgraph)
|
||||
.with_command([
|
||||
"text-completion-ollama",
|
||||
"-p",
|
||||
url.pulsar,
|
||||
"-m",
|
||||
$["ollama-model"],
|
||||
"-i",
|
||||
"non-persistent://tg/request/text-completion-rag",
|
||||
"-o",
|
||||
"non-persistent://tg/response/text-completion-rag",
|
||||
])
|
||||
.with_env_var_secrets(envSecrets)
|
||||
.with_limits("0.5", "128M")
|
||||
.with_reservations("0.1", "128M");
|
||||
|
||||
local containerSet = engine.containers(
|
||||
"text-completion", [ container ]
|
||||
);
|
||||
|
||||
local containerSetRag = engine.containers(
|
||||
"text-completion-rag", [ containerRag ]
|
||||
);
|
||||
|
||||
local service =
|
||||
engine.internalService(containerSet)
|
||||
.with_port(8080, 8080, "metrics");
|
||||
|
||||
local serviceRag =
|
||||
engine.internalService(containerSetRag)
|
||||
.with_port(8080, 8080, "metrics");
|
||||
|
||||
engine.resources([
|
||||
envSecrets,
|
||||
containerSet,
|
||||
containerSetRag,
|
||||
service,
|
||||
serviceRag,
|
||||
])
|
||||
|
||||
},
|
||||
|
|
|
|||
63
templates/components/openai-rag.jsonnet
Normal file
63
templates/components/openai-rag.jsonnet
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
local base = import "base/base.jsonnet";
|
||||
local images = import "values/images.jsonnet";
|
||||
local url = import "values/url.jsonnet";
|
||||
local prompts = import "prompts/mixtral.jsonnet";
|
||||
|
||||
{
|
||||
|
||||
with:: function(key, value)
|
||||
self + {
|
||||
["openai-rag-" + key]:: value,
|
||||
},
|
||||
|
||||
"openai-rag-max-output-tokens":: 4096,
|
||||
"openai-rag-temperature":: 0.0,
|
||||
"openai-rag-model":: "GPT-3.5-Turbo",
|
||||
|
||||
"text-completion-rag" +: {
|
||||
|
||||
create:: function(engine)
|
||||
|
||||
local envSecrets = engine.envSecrets("openai-credentials")
|
||||
.with_env_var("OPENAI_TOKEN", "openai-token");
|
||||
|
||||
local containerRag =
|
||||
engine.container("text-completion-rag")
|
||||
.with_image(images.trustgraph_flow)
|
||||
.with_command([
|
||||
"text-completion-openai",
|
||||
"-p",
|
||||
url.pulsar,
|
||||
"-x",
|
||||
std.toString($["openai-rag-max-output-tokens"]),
|
||||
"-t",
|
||||
"%0.3f" % $["openai-rag-temperature"],
|
||||
"-m",
|
||||
$["openai-rag-model"],
|
||||
"-i",
|
||||
"non-persistent://tg/request/text-completion-rag",
|
||||
"-o",
|
||||
"non-persistent://tg/response/text-completion-rag",
|
||||
])
|
||||
.with_env_var_secrets(envSecrets)
|
||||
.with_limits("0.5", "128M")
|
||||
.with_reservations("0.1", "128M");
|
||||
|
||||
local containerSetRag = engine.containers(
|
||||
"text-completion-rag", [ containerRag ]
|
||||
);
|
||||
|
||||
local serviceRag =
|
||||
engine.internalService(containerSetRag)
|
||||
.with_port(8080, 8080, "metrics");
|
||||
|
||||
engine.resources([
|
||||
envSecrets,
|
||||
containerSetRag,
|
||||
serviceRag,
|
||||
])
|
||||
|
||||
},
|
||||
|
||||
} + prompts
|
||||
|
||||
|
|
@ -5,6 +5,11 @@ local prompts = import "prompts/mixtral.jsonnet";
|
|||
|
||||
{
|
||||
|
||||
with:: function(key, value)
|
||||
self + {
|
||||
["openai-" + key]:: value,
|
||||
},
|
||||
|
||||
"openai-max-output-tokens":: 4096,
|
||||
"openai-temperature":: 0.0,
|
||||
"openai-model":: "GPT-3.5-Turbo",
|
||||
|
|
@ -18,7 +23,7 @@ local prompts = import "prompts/mixtral.jsonnet";
|
|||
|
||||
local container =
|
||||
engine.container("text-completion")
|
||||
.with_image(images.trustgraph)
|
||||
.with_image(images.trustgraph_flow)
|
||||
.with_command([
|
||||
"text-completion-openai",
|
||||
"-p",
|
||||
|
|
@ -34,50 +39,18 @@ local prompts = import "prompts/mixtral.jsonnet";
|
|||
.with_limits("0.5", "128M")
|
||||
.with_reservations("0.1", "128M");
|
||||
|
||||
local containerRag =
|
||||
engine.container("text-completion-rag")
|
||||
.with_image(images.trustgraph)
|
||||
.with_command([
|
||||
"text-completion-openai",
|
||||
"-p",
|
||||
url.pulsar,
|
||||
"-x",
|
||||
std.toString($["openai-max-output-tokens"]),
|
||||
"-t",
|
||||
"%0.3f" % $["openai-temperature"],
|
||||
"-m",
|
||||
$["openai-model"],
|
||||
"-i",
|
||||
"non-persistent://tg/request/text-completion-rag",
|
||||
"-o",
|
||||
"non-persistent://tg/response/text-completion-rag",
|
||||
])
|
||||
.with_env_var_secrets(envSecrets)
|
||||
.with_limits("0.5", "128M")
|
||||
.with_reservations("0.1", "128M");
|
||||
|
||||
local containerSet = engine.containers(
|
||||
"text-completion", [ container ]
|
||||
);
|
||||
|
||||
local containerSetRag = engine.containers(
|
||||
"text-completion-rag", [ containerRag ]
|
||||
);
|
||||
|
||||
local service =
|
||||
engine.internalService(containerSet)
|
||||
.with_port(8080, 8080, "metrics");
|
||||
|
||||
local serviceRag =
|
||||
engine.internalService(containerSetRag)
|
||||
.with_port(8080, 8080, "metrics");
|
||||
|
||||
engine.resources([
|
||||
envSecrets,
|
||||
containerSet,
|
||||
containerSetRag,
|
||||
service,
|
||||
serviceRag,
|
||||
])
|
||||
|
||||
},
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ local cassandra_hosts = "cassandra";
|
|||
|
||||
local container =
|
||||
engine.container("store-graph-embeddings")
|
||||
.with_image(images.trustgraph)
|
||||
.with_image(images.trustgraph_flow)
|
||||
.with_command([
|
||||
"ge-write-pinecone",
|
||||
"-p",
|
||||
|
|
@ -52,7 +52,7 @@ local cassandra_hosts = "cassandra";
|
|||
|
||||
local container =
|
||||
engine.container("query-graph-embeddings")
|
||||
.with_image(images.trustgraph)
|
||||
.with_image(images.trustgraph_flow)
|
||||
.with_command([
|
||||
"ge-query-pinecone",
|
||||
"-p",
|
||||
|
|
@ -87,7 +87,7 @@ local cassandra_hosts = "cassandra";
|
|||
|
||||
local container =
|
||||
engine.container("store-doc-embeddings")
|
||||
.with_image(images.trustgraph)
|
||||
.with_image(images.trustgraph_flow)
|
||||
.with_command([
|
||||
"de-write-pinecone",
|
||||
"-p",
|
||||
|
|
@ -122,7 +122,7 @@ local cassandra_hosts = "cassandra";
|
|||
|
||||
local container =
|
||||
engine.container("query-doc-embeddings")
|
||||
.with_image(images.trustgraph)
|
||||
.with_image(images.trustgraph_flow)
|
||||
.with_command([
|
||||
"de-query-pinecone",
|
||||
"-p",
|
||||
|
|
|
|||
|
|
@ -1,7 +1,3 @@
|
|||
local base = import "base/base.jsonnet";
|
||||
local images = import "values/images.jsonnet";
|
||||
local url = import "values/url.jsonnet";
|
||||
local prompts = import "prompts/mixtral.jsonnet";
|
||||
local default_prompts = import "prompts/default-prompts.jsonnet";
|
||||
|
||||
{
|
||||
|
|
|
|||
|
|
@ -44,7 +44,7 @@ local default_prompts = import "prompts/default-prompts.jsonnet";
|
|||
|
||||
local container =
|
||||
engine.container("prompt")
|
||||
.with_image(images.trustgraph)
|
||||
.with_image(images.trustgraph_flow)
|
||||
.with_command([
|
||||
"prompt-template",
|
||||
"-p",
|
||||
|
|
@ -84,7 +84,7 @@ local default_prompts = import "prompts/default-prompts.jsonnet";
|
|||
|
||||
local container =
|
||||
engine.container("prompt-rag")
|
||||
.with_image(images.trustgraph)
|
||||
.with_image(images.trustgraph_flow)
|
||||
.with_command([
|
||||
"prompt-template",
|
||||
"-p",
|
||||
|
|
|
|||
|
|
@ -2,32 +2,114 @@ local base = import "base/base.jsonnet";
|
|||
local images = import "values/images.jsonnet";
|
||||
local url = import "values/url.jsonnet";
|
||||
|
||||
// This is a Pulsar configuration. Non-standalone mode so we deploy
|
||||
// individual components: bookkeeper, broker and zookeeper.
|
||||
//
|
||||
// This also deploys the TrustGraph 'admin' container which initialises
|
||||
// TrustGraph-specific namespaces etc.
|
||||
|
||||
{
|
||||
|
||||
"pulsar" +: {
|
||||
|
||||
create:: function(engine)
|
||||
|
||||
// local confVolume = engine.volume("pulsar-conf").with_size("2G");
|
||||
local dataVolume = engine.volume("pulsar-data").with_size("20G");
|
||||
// Zookeeper volume
|
||||
local zkVolume = engine.volume("zookeeper").with_size("1G");
|
||||
|
||||
local container =
|
||||
// Zookeeper container
|
||||
local zkContainer =
|
||||
engine.container("zookeeper")
|
||||
.with_image(images.pulsar)
|
||||
.with_command([
|
||||
"bash",
|
||||
"-c",
|
||||
"bin/apply-config-from-env.py conf/zookeeper.conf && bin/generate-zookeeper-config.sh conf/zookeeper.conf && exec bin/pulsar zookeeper"
|
||||
])
|
||||
.with_limits("1", "400M")
|
||||
.with_reservations("0.05", "400M")
|
||||
.with_user("0:1000")
|
||||
.with_volume_mount(zkVolume, "/pulsar/data/zookeeper")
|
||||
.with_environment({
|
||||
"metadataStoreUrl": "zk:zookeeper:2181",
|
||||
"PULSAR_MEM": "-Xms256m -Xmx256m -XX:MaxDirectMemorySize=256m",
|
||||
})
|
||||
.with_port(2181, 2181, "zookeeper")
|
||||
.with_port(2888, 2888, "zookeeper2")
|
||||
.with_port(3888, 3888, "zookeeper3");
|
||||
|
||||
// Pulsar cluster init container
|
||||
local initContainer =
|
||||
engine.container("pulsar-init")
|
||||
.with_image(images.pulsar)
|
||||
.with_command([
|
||||
"bash",
|
||||
"-c",
|
||||
"sleep 10 && bin/pulsar initialize-cluster-metadata --cluster cluster-a --zookeeper zookeeper:2181 --configuration-store zookeeper:2181 --web-service-url http://pulsar:8080 --broker-service-url pulsar://pulsar:6650",
|
||||
])
|
||||
.with_limits("1", "512M")
|
||||
.with_reservations("0.05", "512M")
|
||||
.with_environment({
|
||||
"PULSAR_MEM": "-Xms256m -Xmx256m -XX:MaxDirectMemorySize=256m",
|
||||
});
|
||||
|
||||
|
||||
// Bookkeeper volume
|
||||
local bookieVolume = engine.volume("bookie").with_size("20G");
|
||||
|
||||
// Bookkeeper container
|
||||
local bookieContainer =
|
||||
engine.container("bookie")
|
||||
.with_image(images.pulsar)
|
||||
.with_command([
|
||||
"bash",
|
||||
"-c",
|
||||
"bin/apply-config-from-env.py conf/bookkeeper.conf && exec bin/pulsar bookie"
|
||||
// false ^ causes this to be a 'failure' exit.
|
||||
])
|
||||
.with_limits("1", "800M")
|
||||
.with_reservations("0.1", "800M")
|
||||
.with_user("0:1000")
|
||||
.with_volume_mount(bookieVolume, "/pulsar/data/bookkeeper")
|
||||
.with_environment({
|
||||
"clusterName": "cluster-a",
|
||||
"zkServers": "zookeeper:2181",
|
||||
"bookieId": "bookie",
|
||||
"metadataStoreUri": "metadata-store:zk:zookeeper:2181",
|
||||
"advertisedAddress": "bookie",
|
||||
"BOOKIE_MEM": "-Xms512m -Xmx512m -XX:MaxDirectMemorySize=256m",
|
||||
})
|
||||
.with_port(3181, 3181, "bookie");
|
||||
|
||||
// Pulsar broker, stateless (uses ZK and Bookkeeper for state)
|
||||
local brokerContainer =
|
||||
engine.container("pulsar")
|
||||
.with_image(images.pulsar)
|
||||
.with_command(["bin/pulsar", "standalone"])
|
||||
.with_command([
|
||||
"bash",
|
||||
"-c",
|
||||
"bin/apply-config-from-env.py conf/broker.conf && exec bin/pulsar broker"
|
||||
])
|
||||
.with_limits("1", "800M")
|
||||
.with_reservations("0.1", "800M")
|
||||
.with_environment({
|
||||
"PULSAR_MEM": "-Xms600M -Xmx600M"
|
||||
"metadataStoreUrl": "zk:zookeeper:2181",
|
||||
"zookeeperServers": "zookeeper:2181",
|
||||
"clusterName": "cluster-a",
|
||||
"managedLedgerDefaultEnsembleSize": "1",
|
||||
"managedLedgerDefaultWriteQuorum": "1",
|
||||
"managedLedgerDefaultAckQuorum": "1",
|
||||
"advertisedAddress": "pulsar",
|
||||
"advertisedListeners": "external:pulsar://pulsar:6650,localhost:pulsar://localhost:6650",
|
||||
"PULSAR_MEM": "-Xms512m -Xmx512m -XX:MaxDirectMemorySize=256m",
|
||||
})
|
||||
.with_limits("2.0", "1500M")
|
||||
.with_reservations("1.0", "1500M")
|
||||
// .with_volume_mount(confVolume, "/pulsar/conf")
|
||||
.with_volume_mount(dataVolume, "/pulsar/data")
|
||||
.with_port(6650, 6650, "bookie")
|
||||
.with_port(8080, 8080, "http");
|
||||
.with_port(6650, 6650, "pulsar")
|
||||
.with_port(8080, 8080, "admin");
|
||||
|
||||
// Trustgraph Pulsar initialisation
|
||||
local adminContainer =
|
||||
engine.container("init-pulsar")
|
||||
.with_image(images.trustgraph)
|
||||
engine.container("init-trustgraph")
|
||||
.with_image(images.trustgraph_flow)
|
||||
.with_command([
|
||||
"tg-init-pulsar",
|
||||
"-p",
|
||||
|
|
@ -36,10 +118,32 @@ local url = import "values/url.jsonnet";
|
|||
.with_limits("1", "128M")
|
||||
.with_reservations("0.1", "128M");
|
||||
|
||||
local containerSet = engine.containers(
|
||||
// Container sets
|
||||
local zkContainerSet = engine.containers(
|
||||
"zookeeper",
|
||||
[
|
||||
zkContainer,
|
||||
]
|
||||
);
|
||||
|
||||
local initContainerSet = engine.containers(
|
||||
"init-pulsar",
|
||||
[
|
||||
initContainer,
|
||||
]
|
||||
);
|
||||
|
||||
local bookieContainerSet = engine.containers(
|
||||
"bookie",
|
||||
[
|
||||
bookieContainer,
|
||||
]
|
||||
);
|
||||
|
||||
local brokerContainerSet = engine.containers(
|
||||
"pulsar",
|
||||
[
|
||||
container
|
||||
brokerContainer,
|
||||
]
|
||||
);
|
||||
|
||||
|
|
@ -50,17 +154,35 @@ local url = import "values/url.jsonnet";
|
|||
]
|
||||
);
|
||||
|
||||
local service =
|
||||
engine.service(containerSet)
|
||||
.with_port(6650, 6650, "bookie")
|
||||
.with_port(8080, 8080, "http");
|
||||
// Zookeeper service
|
||||
local zkService =
|
||||
engine.service(zkContainerSet)
|
||||
.with_port(2181, 2181, "zookeeper")
|
||||
.with_port(2888, 2888, "zookeeper2")
|
||||
.with_port(3888, 3888, "zookeeper3");
|
||||
|
||||
// Bookkeeper service
|
||||
local bookieService =
|
||||
engine.service(bookieContainerSet)
|
||||
.with_port(3181, 3181, "bookie");
|
||||
|
||||
// Pulsar broker service
|
||||
local brokerService =
|
||||
engine.service(brokerContainerSet)
|
||||
.with_port(6650, 6650, "pulsar")
|
||||
.with_port(8080, 8080, "admin");
|
||||
|
||||
engine.resources([
|
||||
// confVolume,
|
||||
dataVolume,
|
||||
containerSet,
|
||||
zkVolume,
|
||||
bookieVolume,
|
||||
zkContainerSet,
|
||||
initContainerSet,
|
||||
bookieContainerSet,
|
||||
brokerContainerSet,
|
||||
adminContainerSet,
|
||||
service,
|
||||
zkService,
|
||||
bookieService,
|
||||
brokerService,
|
||||
])
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ qdrant + {
|
|||
|
||||
local container =
|
||||
engine.container("store-graph-embeddings")
|
||||
.with_image(images.trustgraph)
|
||||
.with_image(images.trustgraph_flow)
|
||||
.with_command([
|
||||
"ge-write-qdrant",
|
||||
"-p",
|
||||
|
|
@ -44,7 +44,7 @@ qdrant + {
|
|||
|
||||
local container =
|
||||
engine.container("query-graph-embeddings")
|
||||
.with_image(images.trustgraph)
|
||||
.with_image(images.trustgraph_flow)
|
||||
.with_command([
|
||||
"ge-query-qdrant",
|
||||
"-p",
|
||||
|
|
@ -76,7 +76,7 @@ qdrant + {
|
|||
|
||||
local container =
|
||||
engine.container("store-doc-embeddings")
|
||||
.with_image(images.trustgraph)
|
||||
.with_image(images.trustgraph_flow)
|
||||
.with_command([
|
||||
"de-write-qdrant",
|
||||
"-p",
|
||||
|
|
@ -108,7 +108,7 @@ qdrant + {
|
|||
|
||||
local container =
|
||||
engine.container("query-doc-embeddings")
|
||||
.with_image(images.trustgraph)
|
||||
.with_image(images.trustgraph_flow)
|
||||
.with_command([
|
||||
"de-query-qdrant",
|
||||
"-p",
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
local base = import "base/base.jsonnet";
|
||||
local images = import "values/images.jsonnet";
|
||||
local url = import "values/url.jsonnet";
|
||||
local prompt = import "prompt-template.jsonnet";
|
||||
|
||||
{
|
||||
|
||||
|
|
@ -22,7 +21,7 @@ local prompt = import "prompt-template.jsonnet";
|
|||
|
||||
local container =
|
||||
engine.container("api-gateway")
|
||||
.with_image(images.trustgraph)
|
||||
.with_image(images.trustgraph_flow)
|
||||
.with_command([
|
||||
"api-gateway",
|
||||
"-p",
|
||||
|
|
@ -61,7 +60,7 @@ local prompt = import "prompt-template.jsonnet";
|
|||
|
||||
local container =
|
||||
engine.container("chunker")
|
||||
.with_image(images.trustgraph)
|
||||
.with_image(images.trustgraph_flow)
|
||||
.with_command([
|
||||
"chunker-token",
|
||||
"-p",
|
||||
|
|
@ -95,7 +94,7 @@ local prompt = import "prompt-template.jsonnet";
|
|||
|
||||
local container =
|
||||
engine.container("pdf-decoder")
|
||||
.with_image(images.trustgraph)
|
||||
.with_image(images.trustgraph_flow)
|
||||
.with_command([
|
||||
"pdf-decoder",
|
||||
"-p",
|
||||
|
|
@ -119,43 +118,13 @@ local prompt = import "prompt-template.jsonnet";
|
|||
|
||||
},
|
||||
|
||||
"vectorize" +: {
|
||||
|
||||
create:: function(engine)
|
||||
|
||||
local container =
|
||||
engine.container("vectorize")
|
||||
.with_image(images.trustgraph)
|
||||
.with_command([
|
||||
"embeddings-vectorize",
|
||||
"-p",
|
||||
url.pulsar,
|
||||
])
|
||||
.with_limits("1.0", "512M")
|
||||
.with_reservations("0.5", "512M");
|
||||
|
||||
local containerSet = engine.containers(
|
||||
"vectorize", [ container ]
|
||||
);
|
||||
|
||||
local service =
|
||||
engine.internalService(containerSet)
|
||||
.with_port(8000, 8000, "metrics");
|
||||
|
||||
engine.resources([
|
||||
containerSet,
|
||||
service,
|
||||
])
|
||||
|
||||
},
|
||||
|
||||
"metering" +: {
|
||||
|
||||
create:: function(engine)
|
||||
|
||||
local container =
|
||||
engine.container("metering")
|
||||
.with_image(images.trustgraph)
|
||||
.with_image(images.trustgraph_flow)
|
||||
.with_command([
|
||||
"metering",
|
||||
"-p",
|
||||
|
|
@ -185,7 +154,7 @@ local prompt = import "prompt-template.jsonnet";
|
|||
|
||||
local container =
|
||||
engine.container("metering-rag")
|
||||
.with_image(images.trustgraph)
|
||||
.with_image(images.trustgraph_flow)
|
||||
.with_command([
|
||||
"metering",
|
||||
"-p",
|
||||
|
|
@ -211,5 +180,5 @@ local prompt = import "prompt-template.jsonnet";
|
|||
|
||||
},
|
||||
|
||||
} + prompt
|
||||
}
|
||||
|
||||
|
|
|
|||
74
templates/components/vertexai-rag.jsonnet
Normal file
74
templates/components/vertexai-rag.jsonnet
Normal file
|
|
@ -0,0 +1,74 @@
|
|||
local base = import "base/base.jsonnet";
|
||||
local images = import "values/images.jsonnet";
|
||||
local url = import "values/url.jsonnet";
|
||||
local prompts = import "prompts/mixtral.jsonnet";
|
||||
|
||||
{
|
||||
|
||||
with:: function(key, value)
|
||||
self + {
|
||||
["vertexai-rag-" + key]:: value,
|
||||
},
|
||||
|
||||
"vertexai-rag-model":: "gemini-1.0-pro-001",
|
||||
"vertexai-rag-private-key":: "/vertexai/private.json",
|
||||
"vertexai-rag-region":: "us-central1",
|
||||
"vertexai-rag-max-output-tokens":: 4096,
|
||||
"vertexai-rag-temperature":: 0.0,
|
||||
|
||||
"text-completion-rag" +: {
|
||||
|
||||
create:: function(engine)
|
||||
|
||||
local cfgVol = engine.secretVolume(
|
||||
"vertexai-creds",
|
||||
"./vertexai",
|
||||
{
|
||||
"private.json": importstr "vertexai/private.json",
|
||||
}
|
||||
);
|
||||
|
||||
local container =
|
||||
engine.container("text-completion-rag")
|
||||
.with_image(images.trustgraph_vertexai)
|
||||
.with_command([
|
||||
"text-completion-vertexai",
|
||||
"-p",
|
||||
url.pulsar,
|
||||
"-k",
|
||||
$["vertexai-rag-private-key"],
|
||||
"-r",
|
||||
$["vertexai-rag-region"],
|
||||
"-x",
|
||||
std.toString($["vertexai-rag-max-output-tokens"]),
|
||||
"-t",
|
||||
"%0.3f" % $["vertexai-rag-temperature"],
|
||||
"-m",
|
||||
$["vertexai-rag-model"],
|
||||
"-i",
|
||||
"non-persistent://tg/request/text-completion-rag",
|
||||
"-o",
|
||||
"non-persistent://tg/response/text-completion-rag",
|
||||
])
|
||||
.with_limits("0.5", "256M")
|
||||
.with_reservations("0.1", "256M")
|
||||
.with_volume_mount(cfgVol, "/vertexai");
|
||||
|
||||
local containerSet = engine.containers(
|
||||
"text-completion-rag", [ container ]
|
||||
);
|
||||
|
||||
local service =
|
||||
engine.internalService(containerSet)
|
||||
.with_port(8000, 8000, "metrics");
|
||||
|
||||
engine.resources([
|
||||
cfgVol,
|
||||
containerSet,
|
||||
service,
|
||||
])
|
||||
|
||||
}
|
||||
|
||||
} + prompts
|
||||
|
||||
|
|
@ -5,6 +5,11 @@ local prompts = import "prompts/mixtral.jsonnet";
|
|||
|
||||
{
|
||||
|
||||
with:: function(key, value)
|
||||
self + {
|
||||
["vertexai-" + key]:: value,
|
||||
},
|
||||
|
||||
"vertexai-model":: "gemini-1.0-pro-001",
|
||||
"vertexai-private-key":: "/vertexai/private.json",
|
||||
"vertexai-region":: "us-central1",
|
||||
|
|
@ -25,7 +30,7 @@ local prompts = import "prompts/mixtral.jsonnet";
|
|||
|
||||
local container =
|
||||
engine.container("text-completion")
|
||||
.with_image(images.trustgraph)
|
||||
.with_image(images.trustgraph_vertexai)
|
||||
.with_command([
|
||||
"text-completion-vertexai",
|
||||
"-p",
|
||||
|
|
@ -61,59 +66,5 @@ local prompts = import "prompts/mixtral.jsonnet";
|
|||
|
||||
},
|
||||
|
||||
"text-completion-rag" +: {
|
||||
|
||||
create:: function(engine)
|
||||
|
||||
local cfgVol = engine.secretVolume(
|
||||
"vertexai-creds",
|
||||
"./vertexai",
|
||||
{
|
||||
"private.json": importstr "vertexai/private.json",
|
||||
}
|
||||
);
|
||||
|
||||
local container =
|
||||
engine.container("text-completion-rag")
|
||||
.with_image(images.trustgraph)
|
||||
.with_command([
|
||||
"text-completion-vertexai",
|
||||
"-p",
|
||||
url.pulsar,
|
||||
"-k",
|
||||
$["vertexai-private-key"],
|
||||
"-r",
|
||||
$["vertexai-region"],
|
||||
"-x",
|
||||
std.toString($["vertexai-max-output-tokens"]),
|
||||
"-t",
|
||||
"%0.3f" % $["vertexai-temperature"],
|
||||
"-m",
|
||||
$["vertexai-model"],
|
||||
"-i",
|
||||
"non-persistent://tg/request/text-completion-rag",
|
||||
"-o",
|
||||
"non-persistent://tg/response/text-completion-rag",
|
||||
])
|
||||
.with_limits("0.5", "256M")
|
||||
.with_reservations("0.1", "256M")
|
||||
.with_volume_mount(cfgVol, "/vertexai");
|
||||
|
||||
local containerSet = engine.containers(
|
||||
"text-completion-rag", [ container ]
|
||||
);
|
||||
|
||||
local service =
|
||||
engine.internalService(containerSet)
|
||||
.with_port(8000, 8000, "metrics");
|
||||
|
||||
engine.resources([
|
||||
cfgVol,
|
||||
containerSet,
|
||||
service,
|
||||
])
|
||||
|
||||
}
|
||||
|
||||
} + prompts
|
||||
|
||||
|
|
|
|||
|
|
@ -22,6 +22,8 @@
|
|||
|
||||
with_image:: function(x) self + { image: x },
|
||||
|
||||
with_user:: function(x) self + { user: x },
|
||||
|
||||
with_command:: function(x) self + { command: x },
|
||||
|
||||
with_environment:: function(x) self + {
|
||||
|
|
@ -75,6 +77,10 @@
|
|||
{ command: container.command }
|
||||
else {}) +
|
||||
|
||||
(if std.objectHas(container, "user") then
|
||||
{ user: container.user }
|
||||
else {}) +
|
||||
|
||||
(if ! std.isEmpty(container.environment) then
|
||||
{ environment: container.environment }
|
||||
else {}) +
|
||||
|
|
|
|||
|
|
@ -14,6 +14,8 @@
|
|||
|
||||
with_image:: function(x) self + { image: x },
|
||||
|
||||
with_user:: function(x) self + { user: x },
|
||||
|
||||
with_command:: function(x) self + { command: x },
|
||||
|
||||
with_environment:: function(x) self + {
|
||||
|
|
|
|||
|
|
@ -7,6 +7,8 @@ import logging
|
|||
import os
|
||||
import sys
|
||||
import zipfile
|
||||
import pathlib
|
||||
from io import BytesIO
|
||||
|
||||
logger = logging.getLogger("generate")
|
||||
logging.basicConfig(level=logging.INFO, format='%(message)s')
|
||||
|
|
@ -15,9 +17,13 @@ private_json = "Put your GCP private.json here"
|
|||
|
||||
class Generator:
|
||||
|
||||
def __init__(self, config, base="./templates/", version="0.0.0"):
|
||||
def __init__(
|
||||
self, config, templates="./templates/", resources="./resources",
|
||||
version="0.0.0",
|
||||
):
|
||||
|
||||
self.jsonnet_base = base
|
||||
self.templates = pathlib.Path(templates)
|
||||
self.resources = pathlib.Path(resources)
|
||||
self.config = config
|
||||
self.version = f"\"{version}\"".encode("utf-8")
|
||||
|
||||
|
|
@ -34,25 +40,27 @@ class Generator:
|
|||
path = os.path.join(".", dir, filename)
|
||||
return str(path), self.config
|
||||
|
||||
if filename == "version.jsonnet" and dir == "./templates/values/":
|
||||
if filename == "version.jsonnet" and dir == "templates/values/":
|
||||
path = os.path.join(".", dir, filename)
|
||||
return str(path), self.version
|
||||
|
||||
if dir:
|
||||
candidates = [
|
||||
os.path.join(".", dir, filename),
|
||||
os.path.join(".", filename)
|
||||
self.templates.joinpath(dir, filename),
|
||||
self.templates.joinpath(filename),
|
||||
self.resources.joinpath(dir, filename),
|
||||
self.resources.joinpath(filename),
|
||||
]
|
||||
else:
|
||||
candidates = [
|
||||
os.path.join(".", filename)
|
||||
self.templates.joinpath(filename),
|
||||
]
|
||||
|
||||
try:
|
||||
|
||||
if filename == "vertexai/private.json":
|
||||
|
||||
return candidates[0], private_json.encode("utf-8")
|
||||
return str(candidates[0]), private_json.encode("utf-8")
|
||||
|
||||
for c in candidates:
|
||||
logger.debug("Try: %s", c)
|
||||
|
|
@ -68,73 +76,157 @@ class Generator:
|
|||
|
||||
except:
|
||||
|
||||
path = os.path.join(self.jsonnet_base, filename)
|
||||
path = os.path.join(self.templates, filename)
|
||||
logger.debug("Try: %s", path)
|
||||
with open(path, "rb") as f:
|
||||
logger.debug("Loaded: %s", path)
|
||||
return str(path), f.read()
|
||||
|
||||
class Packager:
|
||||
|
||||
def __init__(self):
|
||||
self.templates = pathlib.Path("./templates")
|
||||
self.resources = pathlib.Path("./resources")
|
||||
|
||||
def process(
|
||||
self, config, version="0.0.0", platform="docker-compose",
|
||||
):
|
||||
|
||||
config = config.encode("utf-8")
|
||||
|
||||
gen = Generator(
|
||||
config, templates=self.templates, resources=self.resources,
|
||||
version=version
|
||||
)
|
||||
|
||||
path = self.templates.joinpath(
|
||||
f"config-to-{platform}.jsonnet"
|
||||
)
|
||||
wrapper = path.read_text()
|
||||
|
||||
processed = gen.process(wrapper)
|
||||
|
||||
return processed
|
||||
|
||||
def generate(self, config, version, platform):
|
||||
|
||||
logger.info(f"Generating for platform={platform} version={version}")
|
||||
|
||||
try:
|
||||
|
||||
if platform in set(["docker-compose", "podman-compose"]):
|
||||
return self.generate_docker_compose(
|
||||
"docker-compose", version, config
|
||||
)
|
||||
elif platform in set(["minikube-k8s", "gcp-k8s"]):
|
||||
return self.generate_k8s(
|
||||
platform, version, config
|
||||
)
|
||||
else:
|
||||
raise RuntimeError("Bad configuration")
|
||||
|
||||
except Exception as e:
|
||||
logging.error(f"Exception: {e}")
|
||||
raise e
|
||||
|
||||
def generate_docker_compose(self, platform, version, config):
|
||||
|
||||
processed = self.process(
|
||||
config, platform=platform, version=version
|
||||
)
|
||||
|
||||
y = yaml.dump(processed)
|
||||
|
||||
mem = BytesIO()
|
||||
|
||||
with zipfile.ZipFile(mem, mode='w') as out:
|
||||
|
||||
def output(name, content):
|
||||
logger.info(f"Adding {name}...")
|
||||
out.writestr(name, content)
|
||||
|
||||
fname = "docker-compose.yaml"
|
||||
|
||||
output(fname, y)
|
||||
|
||||
# Grafana config
|
||||
path = self.resources.joinpath(
|
||||
"grafana/dashboards/dashboard.json"
|
||||
)
|
||||
res = path.read_text()
|
||||
output("grafana/dashboards/dashboard.json", res)
|
||||
|
||||
path = self.resources.joinpath(
|
||||
"grafana/provisioning/dashboard.yml"
|
||||
)
|
||||
res = path.read_text()
|
||||
output("grafana/provisioning/dashboard.yml", res)
|
||||
|
||||
path = self.resources.joinpath(
|
||||
"grafana/provisioning/datasource.yml"
|
||||
)
|
||||
res = path.read_text()
|
||||
output("grafana/provisioning/datasource.yml", res)
|
||||
|
||||
# Prometheus config
|
||||
path = self.resources.joinpath(
|
||||
"prometheus/prometheus.yml"
|
||||
)
|
||||
res = path.read_text()
|
||||
output("prometheus/prometheus.yml", res)
|
||||
|
||||
logger.info("Generation complete.")
|
||||
|
||||
return mem.getvalue()
|
||||
|
||||
def generate_k8s(self, platform, version, config):
|
||||
|
||||
processed = self.process(
|
||||
config, platform=platform, version=version
|
||||
)
|
||||
|
||||
y = yaml.dump(processed)
|
||||
|
||||
mem = BytesIO()
|
||||
|
||||
with zipfile.ZipFile(mem, mode='w') as out:
|
||||
|
||||
def output(name, content):
|
||||
logger.info(f"Adding {name}...")
|
||||
out.writestr(name, content)
|
||||
|
||||
fname = "resources.yaml"
|
||||
|
||||
output(fname, y)
|
||||
|
||||
logger.info("Generation complete.")
|
||||
|
||||
return mem.getvalue()
|
||||
|
||||
def main():
|
||||
|
||||
if len(sys.argv) != 3:
|
||||
if len(sys.argv) != 4:
|
||||
print()
|
||||
print("Usage:")
|
||||
print(" generate <outfile> <version> < input.json")
|
||||
print(" generate <outfile> <version> <platform> < input.json")
|
||||
print()
|
||||
sys.exit(1)
|
||||
|
||||
outfile = sys.argv[1]
|
||||
version = sys.argv[2]
|
||||
platform = sys.argv[3]
|
||||
|
||||
cfg = sys.stdin.read()
|
||||
cfg = json.loads(cfg)
|
||||
|
||||
logger.info(f"Outputting to {outfile}...")
|
||||
|
||||
with zipfile.ZipFile(outfile, mode='w') as out:
|
||||
p = Packager()
|
||||
resp = p.generate(cfg, version, platform)
|
||||
|
||||
def output(name, content):
|
||||
logger.info(f"Adding {name}...")
|
||||
out.writestr(name, content)
|
||||
with open(outfile, "wb") as f:
|
||||
f.write(resp)
|
||||
|
||||
fname = "tg-launch.yaml"
|
||||
|
||||
platform = "docker-compose"
|
||||
|
||||
with open(f"./templates/config-to-{platform}.jsonnet", "r") as f:
|
||||
wrapper = f.read()
|
||||
|
||||
gen = Generator(json.dumps(cfg).encode("utf-8"), version=version)
|
||||
|
||||
processed = gen.process(wrapper)
|
||||
|
||||
y = yaml.dump(processed)
|
||||
|
||||
output(fname, y)
|
||||
|
||||
# Placeholder for the private.json file. Won't put actual credentials
|
||||
# here.
|
||||
output("docker-compose/vertexai/private.json", private_json)
|
||||
|
||||
# Grafana config
|
||||
with open("grafana/dashboards/dashboard.json") as f:
|
||||
output(
|
||||
"docker-compose/grafana/dashboards/dashboard.json", f.read()
|
||||
)
|
||||
|
||||
with open("grafana/provisioning/dashboard.yml") as f:
|
||||
output(
|
||||
"docker-compose/grafana/provisioning/dashboard.yml", f.read()
|
||||
)
|
||||
|
||||
with open("grafana/provisioning/datasource.yml") as f:
|
||||
output(
|
||||
"docker-compose/grafana/provisioning/datasource.yml", f.read()
|
||||
)
|
||||
|
||||
# Prometheus config
|
||||
with open("prometheus/prometheus.yml") as f:
|
||||
output("docker-compose/prometheus/prometheus.yml", f.read())
|
||||
return
|
||||
|
||||
main()
|
||||
|
||||
|
|
|
|||
|
|
@ -87,8 +87,18 @@ def full_config_object(
|
|||
):
|
||||
|
||||
return config_object([
|
||||
graph_store, "pulsar", vector_store, embeddings,
|
||||
"graph-rag", "grafana", "trustgraph", llm, "workbench-ui",
|
||||
"triple-store-" + graph_store,
|
||||
"pulsar",
|
||||
"vector-store-" + vector_store,
|
||||
embeddings,
|
||||
"graph-rag",
|
||||
"grafana",
|
||||
"trustgraph-base",
|
||||
llm,
|
||||
llm + "-rag",
|
||||
"workbench-ui",
|
||||
"prompt-template",
|
||||
"agent-manager-react",
|
||||
])
|
||||
|
||||
def generate_config(
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ local images = import "values/images.jsonnet";
|
|||
engine.container("cassandra")
|
||||
.with_image(images.cassandra)
|
||||
.with_environment({
|
||||
JVM_OPTS: "-Xms300M -Xmx300M",
|
||||
JVM_OPTS: "-Xms300M -Xmx300M -Dcassandra.skip_wait_for_gossip_to_settle=0",
|
||||
})
|
||||
.with_limits("1.0", "1000M")
|
||||
.with_reservations("0.5", "1000M")
|
||||
|
|
|
|||
|
|
@ -7,6 +7,8 @@ local images = import "values/images.jsonnet";
|
|||
|
||||
create:: function(engine)
|
||||
|
||||
local vol = engine.volume("memgraph").with_size("20G");
|
||||
|
||||
local container =
|
||||
engine.container("memgraph")
|
||||
.with_image(images.memgraph_mage)
|
||||
|
|
@ -16,7 +18,8 @@ local images = import "values/images.jsonnet";
|
|||
.with_limits("1.0", "1000M")
|
||||
.with_reservations("0.5", "1000M")
|
||||
.with_port(7474, 7474, "api")
|
||||
.with_port(7687, 7687, "api2");
|
||||
.with_port(7687, 7687, "api2")
|
||||
.with_volume_mount(vol, "/var/lib/memgraph");
|
||||
|
||||
local containerSet = engine.containers(
|
||||
"memgraph", [ container ]
|
||||
|
|
@ -28,6 +31,7 @@ local images = import "values/images.jsonnet";
|
|||
.with_port(7687, 7687, "api2");
|
||||
|
||||
engine.resources([
|
||||
vol,
|
||||
containerSet,
|
||||
service,
|
||||
])
|
||||
|
|
@ -65,4 +69,3 @@ local images = import "values/images.jsonnet";
|
|||
},
|
||||
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -14,12 +14,14 @@ local images = import "values/images.jsonnet";
|
|||
.with_image(images.neo4j)
|
||||
.with_environment({
|
||||
NEO4J_AUTH: "neo4j/password",
|
||||
NEO4J_server_memory_pagecache_size: "512m",
|
||||
NEO4J_server_memory_heap_max__size: "512m",
|
||||
// NEO4J_server_bolt_listen__address: "0.0.0.0:7687",
|
||||
// NEO4J_server_default__listen__address: "0.0.0.0",
|
||||
// NEO4J_server_http_listen__address: "0.0.0.0:7474",
|
||||
})
|
||||
.with_limits("1.0", "768M")
|
||||
.with_reservations("0.5", "768M")
|
||||
.with_limits("1.0", "1536M")
|
||||
.with_reservations("0.5", "1536M")
|
||||
.with_port(7474, 7474, "api")
|
||||
.with_port(7687, 7687, "api2")
|
||||
.with_volume_mount(vol, "/data");
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
local version = import "version.jsonnet";
|
||||
{
|
||||
cassandra: "docker.io/cassandra:4.1.6",
|
||||
neo4j: "docker.io/neo4j:5.22.0-community-bullseye",
|
||||
neo4j: "docker.io/neo4j:5.26.0-community-bullseye",
|
||||
pulsar: "docker.io/apachepulsar/pulsar:3.3.1",
|
||||
pulsar_manager: "docker.io/apachepulsar/pulsar-manager:v0.4.0",
|
||||
etcd: "quay.io/coreos/etcd:v3.5.15",
|
||||
|
|
@ -9,10 +9,14 @@ local version = import "version.jsonnet";
|
|||
milvus: "docker.io/milvusdb/milvus:v2.4.9",
|
||||
prometheus: "docker.io/prom/prometheus:v2.53.2",
|
||||
grafana: "docker.io/grafana/grafana:11.1.4",
|
||||
trustgraph: "docker.io/trustgraph/trustgraph-flow:" + version,
|
||||
trustgraph_base: "docker.io/trustgraph/trustgraph-base:" + version,
|
||||
trustgraph_flow: "docker.io/trustgraph/trustgraph-flow:" + version,
|
||||
trustgraph_bedrock: "docker.io/trustgraph/trustgraph-bedrock:" + version,
|
||||
trustgraph_vertexai: "docker.io/trustgraph/trustgraph-vertexai:" + version,
|
||||
trustgraph_hf: "docker.io/trustgraph/trustgraph-hf:" + version,
|
||||
qdrant: "docker.io/qdrant/qdrant:v1.11.1",
|
||||
memgraph_mage: "docker.io/memgraph/memgraph-mage:1.22-memgraph-2.22",
|
||||
memgraph_lab: "docker.io/memgraph/lab:2.19.1",
|
||||
falkordb: "docker.io/falkordb/falkordb:latest",
|
||||
"workbench-ui": "docker.io/trustgraph/workbench-ui:0.1.6",
|
||||
"workbench-ui": "docker.io/trustgraph/workbench-ui:0.2.4",
|
||||
}
|
||||
|
|
|
|||
|
|
@ -131,6 +131,35 @@ class Api:
|
|||
except:
|
||||
raise ProtocolException(f"Response not formatted correctly")
|
||||
|
||||
def document_rag(self, question):
|
||||
|
||||
# The input consists of a question
|
||||
input = {
|
||||
"query": question
|
||||
}
|
||||
|
||||
url = f"{self.url}document-rag"
|
||||
|
||||
# Invoke the API, input is passed as JSON
|
||||
resp = requests.post(url, json=input)
|
||||
|
||||
# Should be a 200 status code
|
||||
if resp.status_code != 200:
|
||||
raise ProtocolException(f"Status code {resp.status_code}")
|
||||
|
||||
try:
|
||||
# Parse the response as JSON
|
||||
object = resp.json()
|
||||
except:
|
||||
raise ProtocolException(f"Expected JSON response")
|
||||
|
||||
self.check_error(resp)
|
||||
|
||||
try:
|
||||
return object["response"]
|
||||
except:
|
||||
raise ProtocolException(f"Response not formatted correctly")
|
||||
|
||||
def embeddings(self, text):
|
||||
|
||||
# The input consists of a text block
|
||||
|
|
|
|||
|
|
@ -1,11 +1,15 @@
|
|||
|
||||
from pulsar.schema import JsonSchema
|
||||
import pulsar
|
||||
from prometheus_client import Histogram, Info, Counter, Enum
|
||||
import time
|
||||
|
||||
from . base_processor import BaseProcessor
|
||||
from .. exceptions import TooManyRequests
|
||||
|
||||
default_rate_limit_retry = 10
|
||||
default_rate_limit_timeout = 7200
|
||||
|
||||
class Consumer(BaseProcessor):
|
||||
|
||||
def __init__(self, **params):
|
||||
|
|
@ -21,11 +25,18 @@ class Consumer(BaseProcessor):
|
|||
|
||||
super(Consumer, self).__init__(**params)
|
||||
|
||||
input_queue = params.get("input_queue")
|
||||
subscriber = params.get("subscriber")
|
||||
input_schema = params.get("input_schema")
|
||||
self.input_queue = params.get("input_queue")
|
||||
self.subscriber = params.get("subscriber")
|
||||
self.input_schema = params.get("input_schema")
|
||||
|
||||
if input_schema == None:
|
||||
self.rate_limit_retry = params.get(
|
||||
"rate_limit_retry", default_rate_limit_retry
|
||||
)
|
||||
self.rate_limit_timeout = params.get(
|
||||
"rate_limit_timeout", default_rate_limit_timeout
|
||||
)
|
||||
|
||||
if self.input_schema == None:
|
||||
raise RuntimeError("input_schema must be specified")
|
||||
|
||||
if not hasattr(__class__, "request_metric"):
|
||||
|
|
@ -43,17 +54,27 @@ class Consumer(BaseProcessor):
|
|||
'processing_count', 'Processing count', ["status"]
|
||||
)
|
||||
|
||||
if not hasattr(__class__, "rate_limit_metric"):
|
||||
__class__.rate_limit_metric = Counter(
|
||||
'rate_limit_count', 'Rate limit event count',
|
||||
)
|
||||
|
||||
__class__.pubsub_metric.info({
|
||||
"input_queue": input_queue,
|
||||
"subscriber": subscriber,
|
||||
"input_schema": input_schema.__name__,
|
||||
"input_queue": self.input_queue,
|
||||
"subscriber": self.subscriber,
|
||||
"input_schema": self.input_schema.__name__,
|
||||
"rate_limit_retry": str(self.rate_limit_retry),
|
||||
"rate_limit_timeout": str(self.rate_limit_timeout),
|
||||
})
|
||||
|
||||
self.consumer = self.client.subscribe(
|
||||
input_queue, subscriber,
|
||||
schema=JsonSchema(input_schema),
|
||||
self.input_queue, self.subscriber,
|
||||
consumer_type=pulsar.ConsumerType.Shared,
|
||||
schema=JsonSchema(self.input_schema),
|
||||
)
|
||||
|
||||
print("Initialised consumer.", flush=True)
|
||||
|
||||
def run(self):
|
||||
|
||||
__class__.state_metric.state('running')
|
||||
|
|
@ -62,31 +83,61 @@ class Consumer(BaseProcessor):
|
|||
|
||||
msg = self.consumer.receive()
|
||||
|
||||
try:
|
||||
expiry = time.time() + self.rate_limit_timeout
|
||||
|
||||
with __class__.request_metric.time():
|
||||
self.handle(msg)
|
||||
# This loop is for retry on rate-limit / resource limits
|
||||
while True:
|
||||
|
||||
# Acknowledge successful processing of the message
|
||||
self.consumer.acknowledge(msg)
|
||||
if time.time() > expiry:
|
||||
|
||||
__class__.processing_metric.labels(status="success").inc()
|
||||
print("Gave up waiting for rate-limit retry", flush=True)
|
||||
|
||||
except TooManyRequests:
|
||||
self.consumer.negative_acknowledge(msg)
|
||||
print("TooManyRequests: will retry")
|
||||
__class__.processing_metric.labels(status="rate-limit").inc()
|
||||
time.sleep(5)
|
||||
continue
|
||||
# Message failed to be processed, this causes it to
|
||||
# be retried
|
||||
self.consumer.negative_acknowledge(msg)
|
||||
|
||||
__class__.processing_metric.labels(status="error").inc()
|
||||
|
||||
# Break out of retry loop, processes next message
|
||||
break
|
||||
|
||||
try:
|
||||
|
||||
with __class__.request_metric.time():
|
||||
self.handle(msg)
|
||||
|
||||
# Acknowledge successful processing of the message
|
||||
self.consumer.acknowledge(msg)
|
||||
|
||||
__class__.processing_metric.labels(status="success").inc()
|
||||
|
||||
# Break out of retry loop
|
||||
break
|
||||
|
||||
except TooManyRequests:
|
||||
|
||||
print("TooManyRequests: will retry...", flush=True)
|
||||
|
||||
__class__.rate_limit_metric.inc()
|
||||
|
||||
# Sleep
|
||||
time.sleep(self.rate_limit_retry)
|
||||
|
||||
# Contine from retry loop, just causes a reprocessing
|
||||
continue
|
||||
|
||||
except Exception as e:
|
||||
except Exception as e:
|
||||
|
||||
print("Exception:", e, flush=True)
|
||||
print("Exception:", e, flush=True)
|
||||
|
||||
# Message failed to be processed
|
||||
self.consumer.negative_acknowledge(msg)
|
||||
# Message failed to be processed, this causes it to
|
||||
# be retried
|
||||
self.consumer.negative_acknowledge(msg)
|
||||
|
||||
__class__.processing_metric.labels(status="error").inc()
|
||||
__class__.processing_metric.labels(status="error").inc()
|
||||
|
||||
# Break out of retry loop, processes next message
|
||||
break
|
||||
|
||||
@staticmethod
|
||||
def add_args(parser, default_input_queue, default_subscriber):
|
||||
|
|
@ -105,3 +156,17 @@ class Consumer(BaseProcessor):
|
|||
help=f'Queue subscriber name (default: {default_subscriber})'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--rate-limit-retry',
|
||||
type=int,
|
||||
default=default_rate_limit_retry,
|
||||
help=f'Rate limit retry (default: {default_rate_limit_retry})'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--rate-limit-timeout',
|
||||
type=int,
|
||||
default=default_rate_limit_timeout,
|
||||
help=f'Rate limit timeout (default: {default_rate_limit_timeout})'
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,111 +1,46 @@
|
|||
|
||||
from pulsar.schema import JsonSchema
|
||||
import pulsar
|
||||
from prometheus_client import Histogram, Info, Counter, Enum
|
||||
import time
|
||||
|
||||
from . base_processor import BaseProcessor
|
||||
from . consumer import Consumer
|
||||
from .. exceptions import TooManyRequests
|
||||
|
||||
# FIXME: Derive from consumer? And producer?
|
||||
|
||||
class ConsumerProducer(BaseProcessor):
|
||||
class ConsumerProducer(Consumer):
|
||||
|
||||
def __init__(self, **params):
|
||||
|
||||
if not hasattr(__class__, "state_metric"):
|
||||
__class__.state_metric = Enum(
|
||||
'processor_state', 'Processor state',
|
||||
states=['starting', 'running', 'stopped']
|
||||
)
|
||||
__class__.state_metric.state('starting')
|
||||
super(ConsumerProducer, self).__init__(**params)
|
||||
|
||||
__class__.state_metric.state('starting')
|
||||
|
||||
input_queue = params.get("input_queue")
|
||||
output_queue = params.get("output_queue")
|
||||
subscriber = params.get("subscriber")
|
||||
input_schema = params.get("input_schema")
|
||||
output_schema = params.get("output_schema")
|
||||
|
||||
if not hasattr(__class__, "request_metric"):
|
||||
__class__.request_metric = Histogram(
|
||||
'request_latency', 'Request latency (seconds)'
|
||||
)
|
||||
self.output_queue = params.get("output_queue")
|
||||
self.output_schema = params.get("output_schema")
|
||||
|
||||
if not hasattr(__class__, "output_metric"):
|
||||
__class__.output_metric = Counter(
|
||||
'output_count', 'Output items created'
|
||||
)
|
||||
|
||||
if not hasattr(__class__, "pubsub_metric"):
|
||||
__class__.pubsub_metric = Info(
|
||||
'pubsub', 'Pub/sub configuration'
|
||||
)
|
||||
|
||||
if not hasattr(__class__, "processing_metric"):
|
||||
__class__.processing_metric = Counter(
|
||||
'processing_count', 'Processing count', ["status"]
|
||||
)
|
||||
|
||||
__class__.pubsub_metric.info({
|
||||
"input_queue": input_queue,
|
||||
"output_queue": output_queue,
|
||||
"subscriber": subscriber,
|
||||
"input_schema": input_schema.__name__,
|
||||
"output_schema": output_schema.__name__,
|
||||
"input_queue": self.input_queue,
|
||||
"output_queue": self.output_queue,
|
||||
"subscriber": self.subscriber,
|
||||
"input_schema": self.input_schema.__name__,
|
||||
"output_schema": self.output_schema.__name__,
|
||||
"rate_limit_retry": str(self.rate_limit_retry),
|
||||
"rate_limit_timeout": str(self.rate_limit_timeout),
|
||||
})
|
||||
|
||||
super(ConsumerProducer, self).__init__(**params)
|
||||
|
||||
if input_schema == None:
|
||||
raise RuntimeError("input_schema must be specified")
|
||||
|
||||
if output_schema == None:
|
||||
if self.output_schema == None:
|
||||
raise RuntimeError("output_schema must be specified")
|
||||
|
||||
self.producer = self.client.create_producer(
|
||||
topic=output_queue,
|
||||
schema=JsonSchema(output_schema),
|
||||
topic=self.output_queue,
|
||||
schema=JsonSchema(self.output_schema),
|
||||
chunking_enabled=True,
|
||||
)
|
||||
|
||||
self.consumer = self.client.subscribe(
|
||||
input_queue, subscriber,
|
||||
schema=JsonSchema(input_schema),
|
||||
)
|
||||
|
||||
def run(self):
|
||||
|
||||
__class__.state_metric.state('running')
|
||||
|
||||
while True:
|
||||
|
||||
msg = self.consumer.receive()
|
||||
|
||||
try:
|
||||
|
||||
with __class__.request_metric.time():
|
||||
resp = self.handle(msg)
|
||||
|
||||
# Acknowledge successful processing of the message
|
||||
self.consumer.acknowledge(msg)
|
||||
|
||||
__class__.processing_metric.labels(status="success").inc()
|
||||
|
||||
except TooManyRequests:
|
||||
self.consumer.negative_acknowledge(msg)
|
||||
print("TooManyRequests: will retry")
|
||||
__class__.processing_metric.labels(status="rate-limit").inc()
|
||||
time.sleep(5)
|
||||
continue
|
||||
|
||||
except Exception as e:
|
||||
|
||||
print("Exception:", e, flush=True)
|
||||
|
||||
# Message failed to be processed
|
||||
self.consumer.negative_acknowledge(msg)
|
||||
|
||||
__class__.processing_metric.labels(status="error").inc()
|
||||
print("Initialised consumer/producer.")
|
||||
|
||||
def send(self, msg, properties={}):
|
||||
self.producer.send(msg, properties)
|
||||
|
|
@ -117,19 +52,7 @@ class ConsumerProducer(BaseProcessor):
|
|||
default_output_queue,
|
||||
):
|
||||
|
||||
BaseProcessor.add_args(parser)
|
||||
|
||||
parser.add_argument(
|
||||
'-i', '--input-queue',
|
||||
default=default_input_queue,
|
||||
help=f'Input queue (default: {default_input_queue})'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-s', '--subscriber',
|
||||
default=default_subscriber,
|
||||
help=f'Queue subscriber name (default: {default_subscriber})'
|
||||
)
|
||||
Consumer.add_args(parser, default_input_queue, default_subscriber)
|
||||
|
||||
parser.add_argument(
|
||||
'-o', '--output-queue',
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ class Producer(BaseProcessor):
|
|||
self.producer = self.client.create_producer(
|
||||
topic=output_queue,
|
||||
schema=JsonSchema(output_schema),
|
||||
chunking_enabled=True,
|
||||
)
|
||||
|
||||
def send(self, msg, properties={}):
|
||||
|
|
|
|||
|
|
@ -40,8 +40,12 @@ class DocumentEmbeddingsClient(BaseClient):
|
|||
output_schema=DocumentEmbeddingsResponse,
|
||||
)
|
||||
|
||||
def request(self, vectors, limit=10, timeout=300):
|
||||
def request(
|
||||
self, vectors, user="trustgraph", collection="default",
|
||||
limit=10, timeout=300
|
||||
):
|
||||
return self.call(
|
||||
user=user, collection=collection,
|
||||
vectors=vectors, limit=limit, timeout=timeout
|
||||
).documents
|
||||
|
||||
|
|
|
|||
|
|
@ -8,7 +8,3 @@ class LlmError(Exception):
|
|||
class ParseError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -35,14 +35,18 @@ chunk_ingest_queue = topic('chunk-load')
|
|||
|
||||
############################################################################
|
||||
|
||||
# Chunk embeddings are an embeddings associated with a text chunk
|
||||
# Document embeddings are embeddings associated with a chunk
|
||||
|
||||
class ChunkEmbeddings(Record):
|
||||
metadata = Metadata()
|
||||
vectors = Array(Array(Double()))
|
||||
chunk = Bytes()
|
||||
vectors = Array(Array(Double()))
|
||||
|
||||
chunk_embeddings_ingest_queue = topic('chunk-embeddings-load')
|
||||
# This is a 'batching' mechanism for the above data
|
||||
class DocumentEmbeddings(Record):
|
||||
metadata = Metadata()
|
||||
chunks = Array(ChunkEmbeddings())
|
||||
|
||||
document_embeddings_store_queue = topic('document-embeddings-store')
|
||||
|
||||
############################################################################
|
||||
|
||||
|
|
@ -51,6 +55,8 @@ chunk_embeddings_ingest_queue = topic('chunk-embeddings-load')
|
|||
class DocumentEmbeddingsRequest(Record):
|
||||
vectors = Array(Array(Double()))
|
||||
limit = Integer()
|
||||
user = String()
|
||||
collection = String()
|
||||
|
||||
class DocumentEmbeddingsResponse(Record):
|
||||
error = Error()
|
||||
|
|
@ -62,3 +68,4 @@ document_embeddings_request_queue = topic(
|
|||
document_embeddings_response_queue = topic(
|
||||
'doc-embeddings', kind='non-persistent', namespace='response',
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -7,12 +7,31 @@ from . metadata import Metadata
|
|||
|
||||
############################################################################
|
||||
|
||||
# Entity context are an entity associated with textual context
|
||||
|
||||
class EntityContext(Record):
|
||||
entity = Value()
|
||||
context = String()
|
||||
|
||||
# This is a 'batching' mechanism for the above data
|
||||
class EntityContexts(Record):
|
||||
metadata = Metadata()
|
||||
entities = Array(EntityContext())
|
||||
|
||||
entity_contexts_ingest_queue = topic('entity-contexts-load')
|
||||
|
||||
############################################################################
|
||||
|
||||
# Graph embeddings are embeddings associated with a graph entity
|
||||
|
||||
class EntityEmbeddings(Record):
|
||||
entity = Value()
|
||||
vectors = Array(Array(Double()))
|
||||
|
||||
# This is a 'batching' mechanism for the above data
|
||||
class GraphEmbeddings(Record):
|
||||
metadata = Metadata()
|
||||
vectors = Array(Array(Double()))
|
||||
entity = Value()
|
||||
entities = Array(EntityEmbeddings())
|
||||
|
||||
graph_embeddings_store_queue = topic('graph-embeddings-store')
|
||||
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ setuptools.setup(
|
|||
python_requires='>=3.8',
|
||||
download_url = "https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v" + version + ".tar.gz",
|
||||
install_requires=[
|
||||
"trustgraph-base>=0.18,<0.19",
|
||||
"trustgraph-base>=0.21,<0.22",
|
||||
"pulsar-client",
|
||||
"prometheus-client",
|
||||
"boto3",
|
||||
|
|
|
|||
|
|
@ -24,32 +24,48 @@ default_subscriber = module
|
|||
default_model = 'mistral.mistral-large-2407-v1:0'
|
||||
default_temperature = 0.0
|
||||
default_max_output = 2048
|
||||
default_aws_id_key = os.getenv("AWS_ID_KEY", None)
|
||||
default_aws_secret = os.getenv("AWS_SECRET", None)
|
||||
default_aws_region = os.getenv("AWS_REGION", 'us-west-2')
|
||||
|
||||
# Actually, these could all just be None, no need to get environment
|
||||
# variables, as Boto3 would pick all these up if not passed in as args
|
||||
default_access_key_id = os.getenv("AWS_ACCESS_KEY_ID", None)
|
||||
default_secret_access_key = os.getenv("AWS_SECRET_ACCESS_KEY", None)
|
||||
default_session_token = os.getenv("AWS_SESSION_TOKEN", None)
|
||||
default_profile = os.getenv("AWS_PROFILE", None)
|
||||
default_region = os.getenv("AWS_DEFAULT_REGION", None)
|
||||
|
||||
class Processor(ConsumerProducer):
|
||||
|
||||
def __init__(self, **params):
|
||||
|
||||
print(params)
|
||||
|
||||
input_queue = params.get("input_queue", default_input_queue)
|
||||
output_queue = params.get("output_queue", default_output_queue)
|
||||
subscriber = params.get("subscriber", default_subscriber)
|
||||
|
||||
model = params.get("model", default_model)
|
||||
aws_id_key = params.get("aws_id_key", default_aws_id_key)
|
||||
aws_secret = params.get("aws_secret", default_aws_secret)
|
||||
aws_region = params.get("aws_region", default_aws_region)
|
||||
temperature = params.get("temperature", default_temperature)
|
||||
max_output = params.get("max_output", default_max_output)
|
||||
|
||||
if aws_id_key is None:
|
||||
raise RuntimeError("AWS ID not specified")
|
||||
aws_access_key_id = params.get(
|
||||
"aws_access_key_id", default_access_key_id
|
||||
)
|
||||
|
||||
if aws_secret is None:
|
||||
raise RuntimeError("AWS secret not specified")
|
||||
aws_secret_access_key = params.get(
|
||||
"aws_secret_access_key", default_secret_access_key
|
||||
)
|
||||
|
||||
if aws_region is None:
|
||||
raise RuntimeError("AWS region not specified")
|
||||
aws_session_token = params.get(
|
||||
"aws_session_token", default_session_token
|
||||
)
|
||||
|
||||
aws_region = params.get(
|
||||
"aws_region", default_region
|
||||
)
|
||||
|
||||
aws_profile = params.get(
|
||||
"aws_profile", default_profile
|
||||
)
|
||||
|
||||
super(Processor, self).__init__(
|
||||
**params | {
|
||||
|
|
@ -82,9 +98,11 @@ class Processor(ConsumerProducer):
|
|||
self.max_output = max_output
|
||||
|
||||
self.session = boto3.Session(
|
||||
aws_access_key_id=aws_id_key,
|
||||
aws_secret_access_key=aws_secret,
|
||||
region_name=aws_region
|
||||
aws_access_key_id=aws_access_key_id,
|
||||
aws_secret_access_key=aws_secret_access_key,
|
||||
aws_session_token=aws_session_token,
|
||||
profile_name=aws_profile,
|
||||
region_name=aws_region,
|
||||
)
|
||||
|
||||
self.bedrock = self.session.client(service_name='bedrock-runtime')
|
||||
|
|
@ -179,9 +197,6 @@ class Processor(ConsumerProducer):
|
|||
accept = 'application/json'
|
||||
contentType = 'application/json'
|
||||
|
||||
# FIXME: Consider catching request limits and raise TooManyRequests
|
||||
# See https://boto3.amazonaws.com/v1/documentation/api/latest/guide/retries.html
|
||||
|
||||
with __class__.text_completion_metric.time():
|
||||
response = self.bedrock.invoke_model(
|
||||
body=promptbody, modelId=self.model, accept=accept,
|
||||
|
|
@ -243,30 +258,18 @@ class Processor(ConsumerProducer):
|
|||
|
||||
print("Done.", flush=True)
|
||||
|
||||
except self.bedrock.exceptions.ThrottlingException as e:
|
||||
|
||||
# FIXME: Wrong exception, don't know what Bedrock throws
|
||||
# for a rate limit
|
||||
except TooManyRequests:
|
||||
print("Hit rate limit:", e, flush=True)
|
||||
|
||||
print("Send rate limit response...", flush=True)
|
||||
|
||||
r = TextCompletionResponse(
|
||||
error=Error(
|
||||
type = "rate-limit",
|
||||
message = str(e),
|
||||
),
|
||||
response=None,
|
||||
in_token=None,
|
||||
out_token=None,
|
||||
model=None,
|
||||
)
|
||||
|
||||
self.producer.send(r, properties={"id": id})
|
||||
|
||||
self.consumer.acknowledge(msg)
|
||||
# Leave rate limit retries to the base handler
|
||||
raise TooManyRequests()
|
||||
|
||||
except Exception as e:
|
||||
|
||||
# Apart from rate limits, treat all exceptions as unrecoverable
|
||||
|
||||
print(type(e))
|
||||
print(f"Exception: {e}")
|
||||
|
||||
print("Send error response...", flush=True)
|
||||
|
|
@ -299,21 +302,27 @@ class Processor(ConsumerProducer):
|
|||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-z', '--aws-id-key',
|
||||
default=default_aws_id_key,
|
||||
help=f'AWS ID Key'
|
||||
'-z', '--aws-access-key-id',
|
||||
default=default_access_key_id,
|
||||
help=f'AWS access key ID'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-k', '--aws-secret',
|
||||
default=default_aws_secret,
|
||||
help=f'AWS Secret Key'
|
||||
'-k', '--aws-secret-access-key',
|
||||
default=default_secret_access_key,
|
||||
help=f'AWS secret access key'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-r', '--aws-region',
|
||||
default=default_aws_region,
|
||||
help=f'AWS Region'
|
||||
default=default_region,
|
||||
help=f'AWS region'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--aws-profile', '--profile',
|
||||
default=default_profile,
|
||||
help=f'AWS profile name'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ diagnostic utility.
|
|||
import msgpack
|
||||
import sys
|
||||
import argparse
|
||||
import json
|
||||
|
||||
def dump(input_file, action):
|
||||
|
||||
|
|
@ -17,7 +18,7 @@ def dump(input_file, action):
|
|||
unpacker = msgpack.Unpacker(f, raw=False)
|
||||
|
||||
for unpacked in unpacker:
|
||||
print(unpacked)
|
||||
print(json.dumps(unpacked))
|
||||
|
||||
def summary(input_file, action):
|
||||
|
||||
|
|
|
|||
|
|
@ -6,24 +6,23 @@ Connects to the graph query service and dumps all graph edges.
|
|||
|
||||
import argparse
|
||||
import os
|
||||
from trustgraph.clients.triples_query_client import TriplesQueryClient
|
||||
from trustgraph.api import Api
|
||||
|
||||
default_pulsar_host = os.getenv("PULSAR_HOST", 'pulsar://localhost:6650')
|
||||
default_pulsar_api_key = os.getenv("PULSAR_API_KEY", None)
|
||||
default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
|
||||
default_user = 'trustgraph'
|
||||
default_collection = 'default'
|
||||
|
||||
def show_graph(pulsar, user, collection, pulsar_api_key=None):
|
||||
def show_graph(url, user, collection):
|
||||
|
||||
tq = TriplesQueryClient(pulsar_host=pulsar, pulsar_api_key=pulsar_api_key)
|
||||
api = Api(url)
|
||||
|
||||
rows = tq.request(
|
||||
user=user, collection=collection,
|
||||
s=None, p=None, o=None, limit=10_000_000
|
||||
rows = api.triples_query(
|
||||
# user=user, collection=collection,
|
||||
s=None, p=None, o=None, limit=10_000,
|
||||
)
|
||||
|
||||
for row in rows:
|
||||
print(row.s.value, row.p.value, row.o.value)
|
||||
print(row.s, row.p, row.o)
|
||||
|
||||
def main():
|
||||
|
||||
|
|
@ -33,35 +32,30 @@ def main():
|
|||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-p', '--pulsar-host',
|
||||
default=default_pulsar_host,
|
||||
help=f'Pulsar host (default: {default_pulsar_host})',
|
||||
'-u', '--api-url',
|
||||
default=default_url,
|
||||
help=f'API URL (default: {default_url})',
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-u', '--user',
|
||||
'-U', '--user',
|
||||
default=default_user,
|
||||
help=f'User ID (default: {default_user})'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-c', '--collection',
|
||||
'-C', '--collection',
|
||||
default=default_collection,
|
||||
help=f'Collection ID (default: {default_collection})'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--pulsar-api-key',
|
||||
default=default_pulsar_api_key,
|
||||
help=f'Pulsar API key',
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
|
||||
show_graph(
|
||||
pulsar=args.pulsar_host, user=args.user,
|
||||
url=args.api_url,
|
||||
user=args.user,
|
||||
collection=args.collection,
|
||||
pulsar_api_key=args.pulsar_api_key,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -5,38 +5,45 @@ Connects to the graph query service and dumps all graph edges in Turtle
|
|||
format.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
from trustgraph.clients.triples_query_client import TriplesQueryClient
|
||||
import rdflib
|
||||
import io
|
||||
import sys
|
||||
import argparse
|
||||
import os
|
||||
|
||||
default_pulsar_host = os.getenv("PULSAR_HOST", 'pulsar://localhost:6650')
|
||||
default_pulsar_api_key = os.getenv("PULSAR_API_KEY", None)
|
||||
from trustgraph.api import Api, Uri
|
||||
|
||||
def show_graph(pulsar, pulsar_api_key=None):
|
||||
default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
|
||||
default_user = 'trustgraph'
|
||||
default_collection = 'default'
|
||||
|
||||
tq = TriplesQueryClient(pulsar_host=pulsar, pulsar_api_key=pulsar_api_key)
|
||||
def show_graph(url, user, collection):
|
||||
|
||||
rows = tq.request(None, None, None, limit=10_000_000)
|
||||
api = Api(url)
|
||||
|
||||
rows = api.triples_query(
|
||||
s=None, p=None, o=None,
|
||||
limit=10_000)
|
||||
# user=user, collection=collection,
|
||||
|
||||
g = rdflib.Graph()
|
||||
|
||||
for row in rows:
|
||||
|
||||
sv = rdflib.term.URIRef(row.s.value)
|
||||
pv = rdflib.term.URIRef(row.p.value)
|
||||
sv = rdflib.term.URIRef(row.s)
|
||||
pv = rdflib.term.URIRef(row.p)
|
||||
|
||||
if row.o.is_uri:
|
||||
if isinstance(row.o, Uri):
|
||||
|
||||
# Skip malformed URLs with spaces in
|
||||
if " " in row.o.value:
|
||||
if " " in row.o:
|
||||
continue
|
||||
|
||||
ov = rdflib.term.URIRef(row.o.value)
|
||||
ov = rdflib.term.URIRef(row.o)
|
||||
|
||||
else:
|
||||
ov = rdflib.term.Literal(row.o.value)
|
||||
|
||||
ov = rdflib.term.Literal(row.o)
|
||||
|
||||
g.add((sv, pv, ov))
|
||||
|
||||
|
|
@ -57,22 +64,32 @@ def main():
|
|||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-p', '--pulsar-host',
|
||||
default=default_pulsar_host,
|
||||
help=f'Pulsar host (default: {default_pulsar_host})',
|
||||
'-u', '--api-url',
|
||||
default=default_url,
|
||||
help=f'API URL (default: {default_url})',
|
||||
)
|
||||
|
||||
|
||||
parser.add_argument(
|
||||
'--pulsar-api-key',
|
||||
default=default_pulsar_api_key,
|
||||
help=f'Pulsar API key',
|
||||
'-U', '--user',
|
||||
default=default_user,
|
||||
help=f'User ID (default: {default_user})'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-C', '--collection',
|
||||
default=default_collection,
|
||||
help=f'Collection ID (default: {default_collection})'
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
|
||||
show_graph(args.pulsar_host, pulsar_api_key=args.pulsar_api_key)
|
||||
show_graph(
|
||||
url=args.api_url,
|
||||
user=args.user,
|
||||
collection=args.collection
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
|
||||
|
|
|
|||
|
|
@ -1,17 +1,18 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
Uses the GraphRAG service to answer a query
|
||||
Uses the GraphRAG service to answer a question
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import textwrap
|
||||
import uuid
|
||||
import asyncio
|
||||
import json
|
||||
from websockets.asyncio.client import connect
|
||||
|
||||
from trustgraph.clients.agent_client import AgentClient
|
||||
|
||||
default_pulsar_host = os.getenv("PULSAR_HOST", 'pulsar://localhost:6650')
|
||||
default_pulsar_api_key = os.getenv("PULSAR_API_KEY", None)
|
||||
default_url = os.getenv("TRUSTGRAPH_URL", 'ws://localhost:8088/')
|
||||
default_user = 'trustgraph'
|
||||
default_collection = 'default'
|
||||
|
||||
|
|
@ -28,15 +29,18 @@ def output(text, prefix="> ", width=78):
|
|||
)
|
||||
print(out)
|
||||
|
||||
def query(
|
||||
pulsar_host, query, user, collection,
|
||||
plan=None, state=None, verbose=False, pulsar_api_key=None
|
||||
async def question(
|
||||
url, question, user, collection,
|
||||
plan=None, state=None, verbose=False
|
||||
):
|
||||
|
||||
am = AgentClient(pulsar_host=pulsar_host, pulsar_api_key=pulsar_api_key)
|
||||
if not url.endswith("/"):
|
||||
url += "/"
|
||||
|
||||
url = url + "api/v1/socket"
|
||||
|
||||
if verbose:
|
||||
output(wrap(query), "\U00002753 ")
|
||||
output(wrap(question), "\U00002753 ")
|
||||
print()
|
||||
|
||||
def think(x):
|
||||
|
|
@ -49,11 +53,43 @@ def query(
|
|||
output(wrap(x), "\U0001f4a1 ")
|
||||
print()
|
||||
|
||||
resp = am.request(
|
||||
question=query, think=think, observe=observe,
|
||||
)
|
||||
mid = str(uuid.uuid4())
|
||||
|
||||
print(resp)
|
||||
async with connect(url) as ws:
|
||||
|
||||
req = json.dumps({
|
||||
"id": mid,
|
||||
"service": "agent",
|
||||
"request": {
|
||||
"question": question,
|
||||
}
|
||||
|
||||
})
|
||||
|
||||
await ws.send(req)
|
||||
|
||||
while True:
|
||||
|
||||
msg = await ws.recv()
|
||||
|
||||
obj = json.loads(msg)
|
||||
|
||||
if obj["id"] != mid:
|
||||
print("Ignore message")
|
||||
continue
|
||||
|
||||
if "thought" in obj["response"]:
|
||||
think(obj["response"]["thought"])
|
||||
|
||||
if "observation" in obj["response"]:
|
||||
observe(obj["response"]["observation"])
|
||||
|
||||
if "answer" in obj["response"]:
|
||||
print(obj["response"]["answer"])
|
||||
|
||||
if obj["complete"]: break
|
||||
|
||||
await ws.close()
|
||||
|
||||
def main():
|
||||
|
||||
|
|
@ -63,25 +99,25 @@ def main():
|
|||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-p', '--pulsar-host',
|
||||
default=default_pulsar_host,
|
||||
help=f'Pulsar host (default: {default_pulsar_host})',
|
||||
'-u', '--url',
|
||||
default=default_url,
|
||||
help=f'API URL (default: {default_url})',
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-q', '--query',
|
||||
'-q', '--question',
|
||||
required=True,
|
||||
help=f'Query to execute',
|
||||
help=f'Question to answer',
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-u', '--user',
|
||||
'-U', '--user',
|
||||
default=default_user,
|
||||
help=f'User ID (default: {default_user})'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-c', '--collection',
|
||||
'-C', '--collection',
|
||||
default=default_collection,
|
||||
help=f'Collection ID (default: {default_collection})'
|
||||
)
|
||||
|
|
@ -101,26 +137,21 @@ def main():
|
|||
action="store_true",
|
||||
help=f'Output thinking/observations'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--pulsar-api-key',
|
||||
default=default_pulsar_api_key,
|
||||
help=f'Pulsar API key',
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
|
||||
query(
|
||||
pulsar_host=args.pulsar_host,
|
||||
query=args.query,
|
||||
user=args.user,
|
||||
collection=args.collection,
|
||||
plan=args.plan,
|
||||
state=args.state,
|
||||
verbose=args.verbose,
|
||||
pulsar_api_key=args.pulsar_api_key,
|
||||
asyncio.run(
|
||||
question(
|
||||
url=args.url,
|
||||
question=args.question,
|
||||
user=args.user,
|
||||
collection=args.collection,
|
||||
plan=args.plan,
|
||||
state=args.state,
|
||||
verbose=args.verbose,
|
||||
)
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
|
|
|
|||
71
trustgraph-cli/scripts/tg-invoke-document-rag
Executable file
71
trustgraph-cli/scripts/tg-invoke-document-rag
Executable file
|
|
@ -0,0 +1,71 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
Uses the GraphRAG service to answer a question
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
from trustgraph.api import Api
|
||||
|
||||
default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
|
||||
default_user = 'trustgraph'
|
||||
default_collection = 'default'
|
||||
|
||||
def question(url, question, user, collection):
|
||||
|
||||
rag = Api(url)
|
||||
|
||||
# user=user, collection=collection,
|
||||
resp = rag.document_rag(question=question)
|
||||
|
||||
print(resp)
|
||||
|
||||
def main():
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
prog='tg-invoke-document-rag',
|
||||
description=__doc__,
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-u', '--url',
|
||||
default=default_url,
|
||||
help=f'API URL (default: {default_url})',
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-q', '--question',
|
||||
required=True,
|
||||
help=f'Question to answer',
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-U', '--user',
|
||||
default=default_user,
|
||||
help=f'User ID (default: {default_user})'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-C', '--collection',
|
||||
default=default_collection,
|
||||
help=f'Collection ID (default: {default_collection})'
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
|
||||
question(
|
||||
url=args.url,
|
||||
question=args.question,
|
||||
user=args.user,
|
||||
collection=args.collection,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
|
||||
print("Exception:", e, flush=True)
|
||||
|
||||
main()
|
||||
|
||||
71
trustgraph-cli/scripts/tg-invoke-graph-rag
Executable file
71
trustgraph-cli/scripts/tg-invoke-graph-rag
Executable file
|
|
@ -0,0 +1,71 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
Uses the GraphRAG service to answer a question
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
from trustgraph.api import Api
|
||||
|
||||
default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
|
||||
default_user = 'trustgraph'
|
||||
default_collection = 'default'
|
||||
|
||||
def question(url, question, user, collection):
|
||||
|
||||
rag = Api(url)
|
||||
|
||||
# user=user, collection=collection,
|
||||
resp = rag.graph_rag(question=question)
|
||||
|
||||
print(resp)
|
||||
|
||||
def main():
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
prog='tg-invoke-graph-rag',
|
||||
description=__doc__,
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-u', '--url',
|
||||
default=default_url,
|
||||
help=f'API URL (default: {default_url})',
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-q', '--question',
|
||||
required=True,
|
||||
help=f'Question to answer',
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-U', '--user',
|
||||
default=default_user,
|
||||
help=f'User ID (default: {default_user})'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-C', '--collection',
|
||||
default=default_collection,
|
||||
help=f'Collection ID (default: {default_collection})'
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
|
||||
question(
|
||||
url=args.url,
|
||||
question=args.question,
|
||||
user=args.user,
|
||||
collection=args.collection,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
|
||||
print("Exception:", e, flush=True)
|
||||
|
||||
main()
|
||||
|
||||
|
|
@ -8,17 +8,15 @@ and user prompt. Both arguments are required.
|
|||
import argparse
|
||||
import os
|
||||
import json
|
||||
from trustgraph.clients.llm_client import LlmClient
|
||||
from trustgraph.api import Api
|
||||
|
||||
default_pulsar_host = os.getenv("PULSAR_HOST", 'pulsar://localhost:6650')
|
||||
default_pulsar_api_key = os.getenv("PULSAR_API_KEY", None)
|
||||
default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
|
||||
|
||||
def query(url, system, prompt):
|
||||
|
||||
def query(pulsar_host, system, prompt, pulsar_api_key=None):
|
||||
api = Api(url)
|
||||
|
||||
cli = LlmClient(pulsar_host=pulsar_host, pulsar_api_key=pulsar_api_key)
|
||||
|
||||
resp = cli.request(system=system, prompt=prompt)
|
||||
resp = api.text_completion(system=system, prompt=prompt)
|
||||
|
||||
print(resp)
|
||||
|
||||
|
|
@ -30,9 +28,9 @@ def main():
|
|||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-p', '--pulsar-host',
|
||||
default=default_pulsar_host,
|
||||
help=f'Pulsar host (default: {default_pulsar_host})',
|
||||
'-u', '--url',
|
||||
default=default_url,
|
||||
help=f'API URL (default: {default_url})',
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
|
|
@ -46,23 +44,15 @@ def main():
|
|||
nargs=1,
|
||||
help='LLM prompt e.g. What is 2 + 2?',
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--pulsar-api-key',
|
||||
default=default_pulsar_api_key,
|
||||
help=f'Pulsar API key',
|
||||
)
|
||||
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
|
||||
query(
|
||||
pulsar_host=args.pulsar_host,
|
||||
url=args.url,
|
||||
system=args.system[0],
|
||||
prompt=args.prompt[0],
|
||||
pulsar_api_key=args.pulsar_api_key,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
|
|
|
|||
|
|
@ -12,17 +12,15 @@ using key=value arguments on the command line, and these replace
|
|||
import argparse
|
||||
import os
|
||||
import json
|
||||
from trustgraph.clients.prompt_client import PromptClient
|
||||
from trustgraph.api import Api
|
||||
|
||||
default_pulsar_host = os.getenv("PULSAR_HOST", 'pulsar://localhost:6650')
|
||||
default_pulsar_api_key = os.getenv("PULSAR_API_KEY", None)
|
||||
default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
|
||||
|
||||
def query(url, template_id, variables):
|
||||
|
||||
def query(pulsar_host, template_id, variables, pulsar_api_key=None):
|
||||
api = Api(url)
|
||||
|
||||
cli = PromptClient(pulsar_host=pulsar_host, pulsar_api_key=pulsar_api_key)
|
||||
|
||||
resp = cli.request(id=template_id, variables=variables)
|
||||
resp = api.prompt(id=template_id, variables=variables)
|
||||
|
||||
if isinstance(resp, str):
|
||||
print(resp)
|
||||
|
|
@ -37,9 +35,9 @@ def main():
|
|||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-p', '--pulsar-host',
|
||||
default=default_pulsar_host,
|
||||
help=f'Pulsar host (default: {default_pulsar_host})',
|
||||
'-u', '--url',
|
||||
default=default_url,
|
||||
help=f'API URL (default: {default_url})',
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
|
|
@ -57,13 +55,6 @@ def main():
|
|||
specified multiple times''',
|
||||
)
|
||||
|
||||
|
||||
parser.add_argument(
|
||||
'--pulsar-api-key',
|
||||
default=default_pulsar_api_key,
|
||||
help=f'Pulsar API key',
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
variables = {}
|
||||
|
|
@ -79,10 +70,9 @@ specified multiple times''',
|
|||
try:
|
||||
|
||||
query(
|
||||
pulsar_host=args.pulsar_host,
|
||||
url=args.url,
|
||||
template_id=args.id[0],
|
||||
variables=variables,
|
||||
pulsar_api_key=args.pulsar_api_key,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
|
|
|
|||
224
trustgraph-cli/scripts/tg-load-doc-embeds
Executable file
224
trustgraph-cli/scripts/tg-load-doc-embeds
Executable file
|
|
@ -0,0 +1,224 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
"""This utility takes a knowledge core and loads it into a running TrustGraph
|
||||
through the API. The knowledge core should be in msgpack format, which is the
|
||||
default format produce by tg-save-kg-core.
|
||||
"""
|
||||
|
||||
import aiohttp
|
||||
import asyncio
|
||||
import msgpack
|
||||
import json
|
||||
import sys
|
||||
import argparse
|
||||
import os
|
||||
import signal
|
||||
|
||||
class Running:
|
||||
def __init__(self): self.running = True
|
||||
def get(self): return self.running
|
||||
def stop(self): self.running = False
|
||||
|
||||
de_counts = 0
|
||||
|
||||
async def load_de(running, queue, url):
|
||||
|
||||
global de_counts
|
||||
|
||||
async with aiohttp.ClientSession() as session:
|
||||
|
||||
async with session.ws_connect(f"{url}load/document-embeddings") as ws:
|
||||
|
||||
while running.get():
|
||||
|
||||
try:
|
||||
msg = await asyncio.wait_for(queue.get(), 1)
|
||||
|
||||
# End of load
|
||||
if msg is None:
|
||||
break
|
||||
|
||||
except:
|
||||
# Hopefully it's TimeoutError. Annoying to match since
|
||||
# it changed in 3.11.
|
||||
continue
|
||||
|
||||
msg = {
|
||||
"metadata": {
|
||||
"id": msg["m"]["i"],
|
||||
"metadata": msg["m"]["m"],
|
||||
"user": msg["m"]["u"],
|
||||
"collection": msg["m"]["c"],
|
||||
},
|
||||
"chunks": [
|
||||
{
|
||||
"chunk": chunk["c"],
|
||||
"vectors": chunk["v"],
|
||||
}
|
||||
for chunk in msg["c"]
|
||||
],
|
||||
}
|
||||
|
||||
try:
|
||||
await ws.send_json(msg)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
|
||||
de_counts += 1
|
||||
|
||||
async def stats(running):
|
||||
|
||||
global de_counts
|
||||
|
||||
while running.get():
|
||||
|
||||
await asyncio.sleep(2)
|
||||
|
||||
print(
|
||||
f"Graph embeddings: {de_counts:10d}"
|
||||
)
|
||||
|
||||
async def loader(running, de_queue, path, format, user, collection):
|
||||
|
||||
if format == "json":
|
||||
|
||||
raise RuntimeError("Not implemented")
|
||||
|
||||
else:
|
||||
|
||||
with open(path, "rb") as f:
|
||||
|
||||
unpacker = msgpack.Unpacker(f, raw=False)
|
||||
|
||||
while running.get():
|
||||
|
||||
try:
|
||||
unpacked = unpacker.unpack()
|
||||
except:
|
||||
break
|
||||
|
||||
if user:
|
||||
unpacked["metadata"]["user"] = user
|
||||
|
||||
if collection:
|
||||
unpacked["metadata"]["collection"] = collection
|
||||
|
||||
if unpacked[0] == "de":
|
||||
qtype = de_queue
|
||||
|
||||
while running.get():
|
||||
|
||||
try:
|
||||
await asyncio.wait_for(qtype.put(unpacked[1]), 0.5)
|
||||
|
||||
# Successful put message, move on
|
||||
break
|
||||
|
||||
except:
|
||||
# Hopefully it's TimeoutError. Annoying to match since
|
||||
# it changed in 3.11.
|
||||
continue
|
||||
|
||||
if not running.get(): break
|
||||
|
||||
# Put 'None' on end of queue to finish
|
||||
while running.get():
|
||||
|
||||
try:
|
||||
await asyncio.wait_for(de_queue.put(None), 1)
|
||||
|
||||
# Successful put message, move on
|
||||
break
|
||||
|
||||
except:
|
||||
# Hopefully it's TimeoutError. Annoying to match since
|
||||
# it changed in 3.11.
|
||||
continue
|
||||
|
||||
async def run(running, **args):
|
||||
|
||||
# Maxsize on queues reduces back-pressure so tg-load-kg-core doesn't
|
||||
# grow to eat all memory
|
||||
de_q = asyncio.Queue(maxsize=10)
|
||||
|
||||
load_task = asyncio.create_task(
|
||||
loader(
|
||||
running=running,
|
||||
de_queue=de_q,
|
||||
path=args["input_file"], format=args["format"],
|
||||
user=args["user"], collection=args["collection"],
|
||||
)
|
||||
|
||||
)
|
||||
|
||||
de_task = asyncio.create_task(
|
||||
load_de(
|
||||
running=running,
|
||||
queue=de_q, url=args["url"] + "api/v1/"
|
||||
)
|
||||
)
|
||||
|
||||
stats_task = asyncio.create_task(stats(running))
|
||||
|
||||
await de_task
|
||||
|
||||
running.stop()
|
||||
|
||||
await load_task
|
||||
await stats_task
|
||||
|
||||
async def main(running):
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
prog='tg-load-kg-core',
|
||||
description=__doc__,
|
||||
)
|
||||
|
||||
default_url = os.getenv("TRUSTGRAPH_API", "http://localhost:8088/")
|
||||
default_user = "trustgraph"
|
||||
collection = "default"
|
||||
|
||||
parser.add_argument(
|
||||
'-u', '--url',
|
||||
default=default_url,
|
||||
help=f'TrustGraph API URL (default: {default_url})',
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-i', '--input-file',
|
||||
# Make it mandatory, difficult to over-write an existing file
|
||||
required=True,
|
||||
help=f'Output file'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--format',
|
||||
default="msgpack",
|
||||
choices=["msgpack", "json"],
|
||||
help=f'Output format (default: msgpack)',
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--user',
|
||||
help=f'User ID to load as (default: from input)'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--collection',
|
||||
help=f'Collection ID to load as (default: from input)'
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
await run(running, **vars(args))
|
||||
|
||||
running = Running()
|
||||
|
||||
def interrupt(sig, frame):
|
||||
running.stop()
|
||||
print('Interrupt')
|
||||
|
||||
signal.signal(signal.SIGINT, interrupt)
|
||||
|
||||
asyncio.run(main(running))
|
||||
|
||||
|
|
@ -51,8 +51,13 @@ async def load_ge(running, queue, url):
|
|||
"user": msg["m"]["u"],
|
||||
"collection": msg["m"]["c"],
|
||||
},
|
||||
"vectors": msg["v"],
|
||||
"entity": msg["e"],
|
||||
"entities": [
|
||||
{
|
||||
"entity": ent["e"],
|
||||
"vectors": ent["v"],
|
||||
}
|
||||
for ent in msg["e"]
|
||||
],
|
||||
}
|
||||
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -6,21 +6,19 @@ Loads a PDF document into TrustGraph processing.
|
|||
|
||||
import pulsar
|
||||
from pulsar.schema import JsonSchema
|
||||
import base64
|
||||
import hashlib
|
||||
import argparse
|
||||
import os
|
||||
import time
|
||||
import uuid
|
||||
|
||||
from trustgraph.schema import Document, document_ingest_queue
|
||||
from trustgraph.schema import Metadata, Triple, Value
|
||||
from trustgraph.log_level import LogLevel
|
||||
from trustgraph.knowledge import hash, to_uri, Uri
|
||||
from trustgraph.api import Api
|
||||
from trustgraph.knowledge import hash, to_uri
|
||||
from trustgraph.knowledge import PREF_PUBEV, PREF_DOC, PREF_ORG
|
||||
from trustgraph.knowledge import Organization, PublicationEvent
|
||||
from trustgraph.knowledge import DigitalDocument
|
||||
|
||||
default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
|
||||
default_user = 'trustgraph'
|
||||
default_collection = 'default'
|
||||
|
||||
|
|
@ -28,33 +26,13 @@ class Loader:
|
|||
|
||||
def __init__(
|
||||
self,
|
||||
pulsar_host,
|
||||
output_queue,
|
||||
url,
|
||||
user,
|
||||
collection,
|
||||
log_level,
|
||||
metadata,
|
||||
pulsar_api_key=None,
|
||||
):
|
||||
|
||||
if pulsar_api_key:
|
||||
auth = pulsar.AuthenticationToken(pulsar_api_key)
|
||||
self.client = pulsar.Client(
|
||||
pulsar_host,
|
||||
authentication=auth,
|
||||
logger=pulsar.ConsoleLogger(log_level.to_pulsar())
|
||||
)
|
||||
else:
|
||||
self.client = pulsar.Client(
|
||||
pulsar_host,
|
||||
logger=pulsar.ConsoleLogger(log_level.to_pulsar())
|
||||
)
|
||||
|
||||
self.producer = self.client.create_producer(
|
||||
topic=output_queue,
|
||||
schema=JsonSchema(Document),
|
||||
chunking_enabled=True,
|
||||
)
|
||||
|
||||
self.api = Api(url)
|
||||
|
||||
self.user = user
|
||||
self.collection = collection
|
||||
|
|
@ -77,49 +55,18 @@ class Loader:
|
|||
|
||||
id = to_uri(PREF_DOC, id)
|
||||
|
||||
triples = []
|
||||
|
||||
def emit(t):
|
||||
triples.append(t)
|
||||
|
||||
self.metadata.id = id
|
||||
self.metadata.emit(emit)
|
||||
|
||||
r = Document(
|
||||
metadata=Metadata(
|
||||
id=id,
|
||||
metadata=[
|
||||
Triple(
|
||||
s=Value(
|
||||
value=t["s"],
|
||||
is_uri=isinstance(t["s"], Uri)
|
||||
),
|
||||
p=Value(
|
||||
value=t["p"],
|
||||
is_uri=isinstance(t["p"], Uri)
|
||||
),
|
||||
o=Value(
|
||||
value=t["o"],
|
||||
is_uri=isinstance(t["o"], Uri)
|
||||
),
|
||||
)
|
||||
for t in triples
|
||||
],
|
||||
user=self.user,
|
||||
collection=self.collection,
|
||||
),
|
||||
data=base64.b64encode(data),
|
||||
self.api.load_document(
|
||||
document=data, id=id, metadata=self.metadata,
|
||||
# user=self.user,
|
||||
# collection=self.collection,
|
||||
)
|
||||
|
||||
self.producer.send(r)
|
||||
|
||||
print(f"{file}: Loaded successfully.")
|
||||
|
||||
except Exception as e:
|
||||
print(f"{file}: Failed: {str(e)}", flush=True)
|
||||
|
||||
def __del__(self):
|
||||
self.client.close()
|
||||
|
||||
def main():
|
||||
|
||||
|
|
@ -128,36 +75,20 @@ def main():
|
|||
description=__doc__,
|
||||
)
|
||||
|
||||
default_pulsar_host = os.getenv("PULSAR_HOST", 'pulsar://localhost:6650')
|
||||
default_pulsar_api_key = os.getenv("PULSAR_API_KEY", None)
|
||||
default_output_queue = document_ingest_queue
|
||||
|
||||
parser.add_argument(
|
||||
'-p', '--pulsar-host',
|
||||
default=default_pulsar_host,
|
||||
help=f'Pulsar host (default: {default_pulsar_host})',
|
||||
'-u', '--url',
|
||||
default=default_url,
|
||||
help=f'API URL (default: {default_url})',
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--pulsar-api-key',
|
||||
default=default_pulsar_api_key,
|
||||
help=f'Pulsar API key',
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-o', '--output-queue',
|
||||
default=default_output_queue,
|
||||
help=f'Output queue (default: {default_output_queue})'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-u', '--user',
|
||||
'-U', '--user',
|
||||
default=default_user,
|
||||
help=f'User ID (default: {default_user})'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-c', '--collection',
|
||||
'-C', '--collection',
|
||||
default=default_collection,
|
||||
help=f'Collection ID (default: {default_collection})'
|
||||
)
|
||||
|
|
@ -199,7 +130,7 @@ def main():
|
|||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--url', help=f'Document URL'
|
||||
'--document-url', help=f'Document URL'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
|
|
@ -210,14 +141,6 @@ def main():
|
|||
'--identifier', '--id', help=f'Document ID'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-l', '--log-level',
|
||||
type=LogLevel,
|
||||
default=LogLevel.ERROR,
|
||||
choices=list(LogLevel),
|
||||
help=f'Output queue (default: info)'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'files', nargs='+',
|
||||
help=f'File to load'
|
||||
|
|
@ -237,7 +160,7 @@ def main():
|
|||
copyright_holder=args.copyright_holder,
|
||||
copyright_year=args.copyright_year,
|
||||
license=args.license,
|
||||
url=args.url,
|
||||
url=args.document_url,
|
||||
keywords=args.keyword,
|
||||
)
|
||||
|
||||
|
|
@ -255,12 +178,9 @@ def main():
|
|||
)
|
||||
|
||||
p = Loader(
|
||||
pulsar_host=args.pulsar_host,
|
||||
pulsar_api_key=args.pulsar_api_key,
|
||||
output_queue=args.output_queue,
|
||||
url=args.url,
|
||||
user=args.user,
|
||||
collection=args.collection,
|
||||
log_level=args.log_level,
|
||||
metadata=document,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -12,14 +12,13 @@ import os
|
|||
import time
|
||||
import uuid
|
||||
|
||||
from trustgraph.schema import TextDocument, text_ingest_queue
|
||||
from trustgraph.schema import Metadata, Triple, Value
|
||||
from trustgraph.log_level import LogLevel
|
||||
from trustgraph.knowledge import hash, to_uri, Literal, Uri
|
||||
from trustgraph.api import Api
|
||||
from trustgraph.knowledge import hash, to_uri
|
||||
from trustgraph.knowledge import PREF_PUBEV, PREF_DOC, PREF_ORG
|
||||
from trustgraph.knowledge import Organization, PublicationEvent
|
||||
from trustgraph.knowledge import DigitalDocument
|
||||
|
||||
default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
|
||||
default_user = 'trustgraph'
|
||||
default_collection = 'default'
|
||||
|
||||
|
|
@ -27,32 +26,14 @@ class Loader:
|
|||
|
||||
def __init__(
|
||||
self,
|
||||
pulsar_host,
|
||||
output_queue,
|
||||
url,
|
||||
user,
|
||||
collection,
|
||||
log_level,
|
||||
metadata,
|
||||
pulsar_api_key=None,
|
||||
):
|
||||
if pulsar_api_key:
|
||||
auth = pulsar.AuthenticationToken(pulsar_api_key)
|
||||
self.client = pulsar.Client(
|
||||
pulsar_host,
|
||||
authentication=auth,
|
||||
logger=pulsar.ConsoleLogger(log_level.to_pulsar())
|
||||
)
|
||||
else:
|
||||
self.client = pulsar.Client(
|
||||
pulsar_host,
|
||||
logger=pulsar.ConsoleLogger(log_level.to_pulsar())
|
||||
)
|
||||
|
||||
self.producer = self.client.create_producer(
|
||||
topic=output_queue,
|
||||
schema=JsonSchema(TextDocument),
|
||||
chunking_enabled=True,
|
||||
)
|
||||
self.api = Api(url)
|
||||
|
||||
self.user = user
|
||||
self.collection = collection
|
||||
|
|
@ -75,49 +56,18 @@ class Loader:
|
|||
|
||||
id = to_uri(PREF_DOC, id)
|
||||
|
||||
triples = []
|
||||
|
||||
def emit(t):
|
||||
triples.append(t)
|
||||
|
||||
self.metadata.id = id
|
||||
self.metadata.emit(emit)
|
||||
|
||||
r = TextDocument(
|
||||
metadata=Metadata(
|
||||
id=id,
|
||||
metadata=[
|
||||
Triple(
|
||||
s=Value(
|
||||
value=t["s"],
|
||||
is_uri=isinstance(t["s"], Uri)
|
||||
),
|
||||
p=Value(
|
||||
value=t["p"],
|
||||
is_uri=isinstance(t["p"], Uri)
|
||||
),
|
||||
o=Value(
|
||||
value=t["o"],
|
||||
is_uri=isinstance(t["o"], Uri)
|
||||
),
|
||||
)
|
||||
for t in triples
|
||||
],
|
||||
user=self.user,
|
||||
collection=self.collection,
|
||||
),
|
||||
text=data,
|
||||
self.api.load_text(
|
||||
text=data, id=id, metadata=self.metadata,
|
||||
# user=self.user,
|
||||
# collection=self.collection,
|
||||
)
|
||||
|
||||
self.producer.send(r)
|
||||
|
||||
print(f"{file}: Loaded successfully.")
|
||||
|
||||
except Exception as e:
|
||||
print(f"{file}: Failed: {str(e)}", flush=True)
|
||||
|
||||
def __del__(self):
|
||||
self.client.close()
|
||||
|
||||
def main():
|
||||
|
||||
|
|
@ -126,37 +76,20 @@ def main():
|
|||
description=__doc__,
|
||||
)
|
||||
|
||||
default_pulsar_host = os.getenv("PULSAR_HOST", 'pulsar://localhost:6650')
|
||||
default_pulsar_api_key = os.getenv("PULSAR_API_KEY", None)
|
||||
|
||||
default_output_queue = text_ingest_queue
|
||||
|
||||
parser.add_argument(
|
||||
'-p', '--pulsar-host',
|
||||
default=default_pulsar_host,
|
||||
help=f'Pulsar host (default: {default_pulsar_host})',
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--pulsar-api-key',
|
||||
default=default_pulsar_api_key,
|
||||
help=f'Pulsar API key',
|
||||
'-u', '--url',
|
||||
default=default_url,
|
||||
help=f'API URL (default: {default_url})',
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-o', '--output-queue',
|
||||
default=default_output_queue,
|
||||
help=f'Output queue (default: {default_output_queue})'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-u', '--user',
|
||||
'-U', '--user',
|
||||
default=default_user,
|
||||
help=f'User ID (default: {default_user})'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-c', '--collection',
|
||||
'-C', '--collection',
|
||||
default=default_collection,
|
||||
help=f'Collection ID (default: {default_collection})'
|
||||
)
|
||||
|
|
@ -198,7 +131,7 @@ def main():
|
|||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--url', help=f'Document URL'
|
||||
'--document-url', help=f'Document URL'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
|
|
@ -209,14 +142,6 @@ def main():
|
|||
'--identifier', '--id', help=f'Document ID'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-l', '--log-level',
|
||||
type=LogLevel,
|
||||
default=LogLevel.ERROR,
|
||||
choices=list(LogLevel),
|
||||
help=f'Output queue (default: info)'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'files', nargs='+',
|
||||
help=f'File to load'
|
||||
|
|
@ -236,7 +161,7 @@ def main():
|
|||
copyright_holder=args.copyright_holder,
|
||||
copyright_year=args.copyright_year,
|
||||
license=args.license,
|
||||
url=args.url,
|
||||
url=args.document_url,
|
||||
keywords=args.keyword,
|
||||
)
|
||||
|
||||
|
|
@ -254,12 +179,9 @@ def main():
|
|||
)
|
||||
|
||||
p = Loader(
|
||||
pulsar_host=args.pulsar_host,
|
||||
pulsar_api_key=args.pulsar_api_key,
|
||||
output_queue=args.output_queue,
|
||||
url=args.url,
|
||||
user=args.user,
|
||||
collection=args.collection,
|
||||
log_level=args.log_level,
|
||||
metadata=document,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,76 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
Uses the Document RAG service to answer a query
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
from trustgraph.clients.document_rag_client import DocumentRagClient
|
||||
|
||||
default_pulsar_host = os.getenv("PULSAR_HOST", 'pulsar://localhost:6650')
|
||||
default_pulsar_api_key = os.getenv("PULSAR_API_KEY", None)
|
||||
|
||||
default_user = 'trustgraph'
|
||||
default_collection = 'default'
|
||||
|
||||
def query(pulsar_host, query, user, collection, pulsar_api_key=None):
|
||||
|
||||
rag = DocumentRagClient(pulsar_host=pulsar_host, pulsar_api_key=pulsar_api_key)
|
||||
resp = rag.request(user=user, collection=collection, query=query)
|
||||
print(resp)
|
||||
|
||||
def main():
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
prog='tg-query-document-rag',
|
||||
description=__doc__,
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-p', '--pulsar-host',
|
||||
default=default_pulsar_host,
|
||||
help=f'Pulsar host (default: {default_pulsar_host})',
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--pulsar-api-key',
|
||||
default=default_pulsar_api_key,
|
||||
help=f'Pulsar API key',
|
||||
)
|
||||
parser.add_argument(
|
||||
'-q', '--query',
|
||||
required=True,
|
||||
help=f'Query to execute',
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-u', '--user',
|
||||
default=default_user,
|
||||
help=f'User ID (default: {default_user})'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-c', '--collection',
|
||||
default=default_collection,
|
||||
help=f'Collection ID (default: {default_collection})'
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
|
||||
query(
|
||||
pulsar_host=args.pulsar_host,
|
||||
pulsar_api_key=args.pulsar_api_key,
|
||||
query=args.query,
|
||||
user=args.user,
|
||||
collection=args.collection,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
|
||||
print("Exception:", e, flush=True)
|
||||
|
||||
main()
|
||||
|
||||
|
|
@ -1,77 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
Uses the GraphRAG service to answer a query
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
from trustgraph.clients.graph_rag_client import GraphRagClient
|
||||
|
||||
default_pulsar_host = os.getenv("PULSAR_HOST", 'pulsar://localhost:6650')
|
||||
default_pulsar_api_key = os.getenv("PULSAR_API_KEY", None)
|
||||
|
||||
default_user = 'trustgraph'
|
||||
default_collection = 'default'
|
||||
|
||||
def query(pulsar_host, query, user, collection, pulsar_api_key=None):
|
||||
|
||||
rag = GraphRagClient(pulsar_host=pulsar_host, pulsar_api_key=pulsar_api_key)
|
||||
resp = rag.request(user=user, collection=collection, query=query)
|
||||
print(resp)
|
||||
|
||||
def main():
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
prog='tg-graph-query-rag',
|
||||
description=__doc__,
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-p', '--pulsar-host',
|
||||
default=default_pulsar_host,
|
||||
help=f'Pulsar host (default: {default_pulsar_host})',
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--pulsar-api-key',
|
||||
default=default_pulsar_api_key,
|
||||
help=f'Pulsar API key',
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-q', '--query',
|
||||
required=True,
|
||||
help=f'Query to execute',
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-u', '--user',
|
||||
default=default_user,
|
||||
help=f'User ID (default: {default_user})'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-c', '--collection',
|
||||
default=default_collection,
|
||||
help=f'Collection ID (default: {default_collection})'
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
|
||||
query(
|
||||
pulsar_host=args.pulsar_host,
|
||||
pulsar_api_key=args.pulsar_api_key,
|
||||
query=args.query,
|
||||
user=args.user,
|
||||
collection=args.collection,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
|
||||
print("Exception:", e, flush=True)
|
||||
|
||||
main()
|
||||
|
||||
198
trustgraph-cli/scripts/tg-save-doc-embeds
Executable file
198
trustgraph-cli/scripts/tg-save-doc-embeds
Executable file
|
|
@ -0,0 +1,198 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
This utility connects to a running TrustGraph through the API and creates
|
||||
a knowledge core from the data streaming through the processing queues.
|
||||
For completeness of data, tg-save-kg-core should be initiated before data
|
||||
loading takes place. The default output format, msgpack should be used.
|
||||
JSON output format is also available - msgpack produces a more compact
|
||||
representation, which is also more performant to load.
|
||||
"""
|
||||
|
||||
import aiohttp
|
||||
import asyncio
|
||||
import msgpack
|
||||
import json
|
||||
import sys
|
||||
import argparse
|
||||
import os
|
||||
import signal
|
||||
|
||||
class Running:
|
||||
def __init__(self): self.running = True
|
||||
def get(self): return self.running
|
||||
def stop(self): self.running = False
|
||||
|
||||
async def fetch_de(running, queue, user, collection, url):
|
||||
|
||||
async with aiohttp.ClientSession() as session:
|
||||
|
||||
de_url = f"{url}stream/document-embeddings"
|
||||
|
||||
async with session.ws_connect(de_url) as ws:
|
||||
|
||||
while running.get():
|
||||
|
||||
try:
|
||||
msg = await asyncio.wait_for(ws.receive(), 1)
|
||||
except:
|
||||
continue
|
||||
|
||||
if msg.type == aiohttp.WSMsgType.TEXT:
|
||||
|
||||
data = msg.json()
|
||||
|
||||
if user:
|
||||
if data["metadata"]["user"] != user:
|
||||
continue
|
||||
|
||||
if collection:
|
||||
if data["metadata"]["collection"] != collection:
|
||||
continue
|
||||
|
||||
await queue.put([
|
||||
"de",
|
||||
{
|
||||
"m": {
|
||||
"i": data["metadata"]["id"],
|
||||
"m": data["metadata"]["metadata"],
|
||||
"u": data["metadata"]["user"],
|
||||
"c": data["metadata"]["collection"],
|
||||
},
|
||||
"c": [
|
||||
{
|
||||
"c": chunk["chunk"],
|
||||
"v": chunk["vectors"],
|
||||
}
|
||||
for chunk in data["chunks"]
|
||||
]
|
||||
}
|
||||
])
|
||||
if msg.type == aiohttp.WSMsgType.ERROR:
|
||||
print("Error")
|
||||
break
|
||||
|
||||
de_counts = 0
|
||||
|
||||
async def stats(running):
|
||||
|
||||
global t_counts
|
||||
global de_counts
|
||||
|
||||
while running.get():
|
||||
|
||||
await asyncio.sleep(2)
|
||||
|
||||
print(
|
||||
f"Document embeddings: {de_counts:10d}"
|
||||
)
|
||||
|
||||
async def output(running, queue, path, format):
|
||||
|
||||
global t_counts
|
||||
global de_counts
|
||||
|
||||
with open(path, "wb") as f:
|
||||
|
||||
while running.get():
|
||||
|
||||
try:
|
||||
msg = await asyncio.wait_for(queue.get(), 0.5)
|
||||
except:
|
||||
# Hopefully it's TimeoutError. Annoying to match since
|
||||
# it changed in 3.11.
|
||||
continue
|
||||
|
||||
if format == "msgpack":
|
||||
f.write(msgpack.packb(msg, use_bin_type=True))
|
||||
else:
|
||||
f.write(json.dumps(msg).encode("utf-8"))
|
||||
|
||||
if msg[0] == "de":
|
||||
de_counts += 1
|
||||
|
||||
print("Output file closed")
|
||||
|
||||
async def run(running, **args):
|
||||
|
||||
q = asyncio.Queue()
|
||||
|
||||
de_task = asyncio.create_task(
|
||||
fetch_de(
|
||||
running=running,
|
||||
queue=q, user=args["user"], collection=args["collection"],
|
||||
url=args["url"] + "api/v1/"
|
||||
)
|
||||
)
|
||||
|
||||
output_task = asyncio.create_task(
|
||||
output(
|
||||
running=running, queue=q,
|
||||
path=args["output_file"], format=args["format"],
|
||||
)
|
||||
|
||||
)
|
||||
|
||||
stats_task = asyncio.create_task(stats(running))
|
||||
|
||||
await output_task
|
||||
await de_task
|
||||
await stats_task
|
||||
|
||||
print("Exiting")
|
||||
|
||||
async def main(running):
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
prog='tg-save-kg-core',
|
||||
description=__doc__,
|
||||
)
|
||||
|
||||
default_url = os.getenv("TRUSTGRAPH_API", "http://localhost:8088/")
|
||||
default_user = "trustgraph"
|
||||
collection = "default"
|
||||
|
||||
parser.add_argument(
|
||||
'-u', '--url',
|
||||
default=default_url,
|
||||
help=f'TrustGraph API URL (default: {default_url})',
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-o', '--output-file',
|
||||
# Make it mandatory, difficult to over-write an existing file
|
||||
required=True,
|
||||
help=f'Output file'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--format',
|
||||
default="msgpack",
|
||||
choices=["msgpack", "json"],
|
||||
help=f'Output format (default: msgpack)',
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--user',
|
||||
help=f'User ID to filter on (default: no filter)'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--collection',
|
||||
help=f'Collection ID to filter on (default: no filter)'
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
await run(running, **vars(args))
|
||||
|
||||
running = Running()
|
||||
|
||||
def interrupt(sig, frame):
|
||||
running.stop()
|
||||
print('Interrupt')
|
||||
|
||||
signal.signal(signal.SIGINT, interrupt)
|
||||
|
||||
asyncio.run(main(running))
|
||||
|
||||
|
|
@ -57,8 +57,13 @@ async def fetch_ge(running, queue, user, collection, url):
|
|||
"u": data["metadata"]["user"],
|
||||
"c": data["metadata"]["collection"],
|
||||
},
|
||||
"v": data["vectors"],
|
||||
"e": data["entity"],
|
||||
"e": [
|
||||
{
|
||||
"e": ent["entity"],
|
||||
"v": ent["vectors"],
|
||||
}
|
||||
for ent in data["entities"]
|
||||
]
|
||||
}
|
||||
])
|
||||
if msg.type == aiohttp.WSMsgType.ERROR:
|
||||
|
|
|
|||
|
|
@ -34,29 +34,33 @@ setuptools.setup(
|
|||
python_requires='>=3.8',
|
||||
download_url = "https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v" + version + ".tar.gz",
|
||||
install_requires=[
|
||||
"trustgraph-base>=0.18,<0.19",
|
||||
"trustgraph-base>=0.21,<0.22",
|
||||
"requests",
|
||||
"pulsar-client",
|
||||
"aiohttp",
|
||||
"rdflib",
|
||||
"tabulate",
|
||||
"msgpack",
|
||||
"websockets",
|
||||
],
|
||||
scripts=[
|
||||
"scripts/tg-dump-msgpack",
|
||||
"scripts/tg-graph-show",
|
||||
"scripts/tg-graph-to-turtle",
|
||||
"scripts/tg-init-pulsar",
|
||||
"scripts/tg-init-pulsar-manager",
|
||||
"scripts/tg-invoke-agent",
|
||||
"scripts/tg-invoke-document-rag",
|
||||
"scripts/tg-invoke-graph-rag",
|
||||
"scripts/tg-invoke-llm",
|
||||
"scripts/tg-invoke-prompt",
|
||||
"scripts/tg-load-kg-core",
|
||||
"scripts/tg-load-doc-embeds",
|
||||
"scripts/tg-load-pdf",
|
||||
"scripts/tg-load-text",
|
||||
"scripts/tg-load-turtle",
|
||||
"scripts/tg-query-document-rag",
|
||||
"scripts/tg-query-graph-rag",
|
||||
"scripts/tg-init-pulsar",
|
||||
"scripts/tg-processor-state",
|
||||
"scripts/tg-invoke-agent",
|
||||
"scripts/tg-invoke-prompt",
|
||||
"scripts/tg-invoke-llm",
|
||||
"scripts/tg-save-kg-core",
|
||||
"scripts/tg-load-kg-core",
|
||||
"scripts/tg-dump-msgpack",
|
||||
"scripts/tg-save-doc-embeds",
|
||||
]
|
||||
)
|
||||
|
|
|
|||
|
|
@ -34,8 +34,8 @@ setuptools.setup(
|
|||
python_requires='>=3.8',
|
||||
download_url = "https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v" + version + ".tar.gz",
|
||||
install_requires=[
|
||||
"trustgraph-base>=0.18,<0.19",
|
||||
"trustgraph-flow>=0.18,<0.19",
|
||||
"trustgraph-base>=0.21,<0.22",
|
||||
"trustgraph-flow>=0.21,<0.22",
|
||||
"torch",
|
||||
"urllib3",
|
||||
"transformers",
|
||||
|
|
|
|||
6
trustgraph-flow/scripts/document-embeddings
Executable file
6
trustgraph-flow/scripts/document-embeddings
Executable file
|
|
@ -0,0 +1,6 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
from trustgraph.embeddings.document_embeddings import run
|
||||
|
||||
run()
|
||||
|
||||
6
trustgraph-flow/scripts/embeddings-fastembed
Executable file
6
trustgraph-flow/scripts/embeddings-fastembed
Executable file
|
|
@ -0,0 +1,6 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
from trustgraph.embeddings.fastembed import run
|
||||
|
||||
run()
|
||||
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
from trustgraph.embeddings.vectorize import run
|
||||
|
||||
run()
|
||||
|
||||
6
trustgraph-flow/scripts/graph-embeddings
Executable file
6
trustgraph-flow/scripts/graph-embeddings
Executable file
|
|
@ -0,0 +1,6 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
from trustgraph.embeddings.graph_embeddings import run
|
||||
|
||||
run()
|
||||
|
||||
|
|
@ -34,58 +34,63 @@ setuptools.setup(
|
|||
python_requires='>=3.8',
|
||||
download_url = "https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v" + version + ".tar.gz",
|
||||
install_requires=[
|
||||
"trustgraph-base>=0.18,<0.19",
|
||||
"urllib3",
|
||||
"rdflib",
|
||||
"pymilvus",
|
||||
"langchain",
|
||||
"langchain-core",
|
||||
"langchain-text-splitters",
|
||||
"langchain-community",
|
||||
"requests",
|
||||
"cassandra-driver",
|
||||
"pulsar-client",
|
||||
"pypdf",
|
||||
"qdrant-client",
|
||||
"tabulate",
|
||||
"trustgraph-base>=0.21,<0.22",
|
||||
"aiohttp",
|
||||
"anthropic",
|
||||
"pyyaml",
|
||||
"prometheus-client",
|
||||
"cassandra-driver",
|
||||
"cohere",
|
||||
"openai",
|
||||
"neo4j",
|
||||
"tiktoken",
|
||||
"cryptography",
|
||||
"falkordb",
|
||||
"fastembed",
|
||||
"google-generativeai",
|
||||
"ibis",
|
||||
"jsonschema",
|
||||
"aiohttp",
|
||||
"langchain",
|
||||
"langchain-community",
|
||||
"langchain-core",
|
||||
"langchain-text-splitters",
|
||||
"neo4j",
|
||||
"ollama",
|
||||
"openai",
|
||||
"pinecone[grpc]",
|
||||
"falkordb",
|
||||
"prometheus-client",
|
||||
"pulsar-client",
|
||||
"pymilvus",
|
||||
"pypdf",
|
||||
"pyyaml",
|
||||
"qdrant-client",
|
||||
"rdflib",
|
||||
"requests",
|
||||
"tabulate",
|
||||
"tiktoken",
|
||||
"urllib3",
|
||||
],
|
||||
scripts=[
|
||||
"scripts/api-gateway",
|
||||
"scripts/agent-manager-react",
|
||||
"scripts/api-gateway",
|
||||
"scripts/chunker-recursive",
|
||||
"scripts/chunker-token",
|
||||
"scripts/de-query-milvus",
|
||||
"scripts/de-query-qdrant",
|
||||
"scripts/de-query-pinecone",
|
||||
"scripts/de-query-qdrant",
|
||||
"scripts/de-write-milvus",
|
||||
"scripts/de-write-qdrant",
|
||||
"scripts/de-write-pinecone",
|
||||
"scripts/de-write-qdrant",
|
||||
"scripts/document-embeddings",
|
||||
"scripts/document-rag",
|
||||
"scripts/embeddings-ollama",
|
||||
"scripts/embeddings-vectorize",
|
||||
"scripts/embeddings-fastembed",
|
||||
"scripts/ge-query-milvus",
|
||||
"scripts/ge-query-pinecone",
|
||||
"scripts/ge-query-qdrant",
|
||||
"scripts/ge-write-milvus",
|
||||
"scripts/ge-write-pinecone",
|
||||
"scripts/ge-write-qdrant",
|
||||
"scripts/graph-embeddings",
|
||||
"scripts/graph-rag",
|
||||
"scripts/kg-extract-definitions",
|
||||
"scripts/kg-extract-topics",
|
||||
"scripts/kg-extract-relationships",
|
||||
"scripts/kg-extract-topics",
|
||||
"scripts/metering",
|
||||
"scripts/object-extract-row",
|
||||
"scripts/oe-write-milvus",
|
||||
|
|
@ -103,13 +108,13 @@ setuptools.setup(
|
|||
"scripts/text-completion-ollama",
|
||||
"scripts/text-completion-openai",
|
||||
"scripts/triples-query-cassandra",
|
||||
"scripts/triples-query-neo4j",
|
||||
"scripts/triples-query-memgraph",
|
||||
"scripts/triples-query-falkordb",
|
||||
"scripts/triples-query-memgraph",
|
||||
"scripts/triples-query-neo4j",
|
||||
"scripts/triples-write-cassandra",
|
||||
"scripts/triples-write-neo4j",
|
||||
"scripts/triples-write-memgraph",
|
||||
"scripts/triples-write-falkordb",
|
||||
"scripts/triples-write-memgraph",
|
||||
"scripts/triples-write-neo4j",
|
||||
"scripts/wikipedia-lookup",
|
||||
]
|
||||
)
|
||||
|
|
|
|||
|
|
@ -14,8 +14,6 @@ from ... schema import AgentRequest, AgentResponse, AgentStep
|
|||
from ... schema import agent_request_queue, agent_response_queue
|
||||
from ... schema import prompt_request_queue as pr_request_queue
|
||||
from ... schema import prompt_response_queue as pr_response_queue
|
||||
from ... schema import text_completion_request_queue as tc_request_queue
|
||||
from ... schema import text_completion_response_queue as tc_response_queue
|
||||
from ... schema import graph_rag_request_queue as gr_request_queue
|
||||
from ... schema import graph_rag_response_queue as gr_response_queue
|
||||
from ... clients.prompt_client import PromptClient
|
||||
|
|
@ -133,12 +131,6 @@ class Processor(ConsumerProducer):
|
|||
prompt_response_queue = params.get(
|
||||
"prompt_response_queue", pr_response_queue
|
||||
)
|
||||
text_completion_request_queue = params.get(
|
||||
"text_completion_request_queue", tc_request_queue
|
||||
)
|
||||
text_completion_response_queue = params.get(
|
||||
"text_completion_response_queue", tc_response_queue
|
||||
)
|
||||
graph_rag_request_queue = params.get(
|
||||
"graph_rag_request_queue", gr_request_queue
|
||||
)
|
||||
|
|
@ -155,8 +147,6 @@ class Processor(ConsumerProducer):
|
|||
"output_schema": AgentResponse,
|
||||
"prompt_request_queue": prompt_request_queue,
|
||||
"prompt_response_queue": prompt_response_queue,
|
||||
"text_completion_request_queue": tc_request_queue,
|
||||
"text_completion_response_queue": tc_response_queue,
|
||||
"graph_rag_request_queue": gr_request_queue,
|
||||
"graph_rag_response_queue": gr_response_queue,
|
||||
}
|
||||
|
|
@ -170,14 +160,6 @@ class Processor(ConsumerProducer):
|
|||
pulsar_api_key=self.pulsar_api_key,
|
||||
)
|
||||
|
||||
self.llm = LlmClient(
|
||||
subscriber=subscriber,
|
||||
input_queue=text_completion_request_queue,
|
||||
output_queue=text_completion_response_queue,
|
||||
pulsar_host = self.pulsar_host,
|
||||
pulsar_api_key=self.pulsar_api_key,
|
||||
)
|
||||
|
||||
self.graph_rag = GraphRagClient(
|
||||
subscriber=subscriber,
|
||||
input_queue=graph_rag_request_queue,
|
||||
|
|
@ -343,18 +325,6 @@ class Processor(ConsumerProducer):
|
|||
help=f'Prompt response queue (default: {pr_response_queue})',
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--text-completion-request-queue',
|
||||
default=tc_request_queue,
|
||||
help=f'Text completion request queue (default: {tc_request_queue})',
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--text-completion-response-queue',
|
||||
default=tc_response_queue,
|
||||
help=f'Text completion response queue (default: {tc_response_queue})',
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--graph-rag-request-queue',
|
||||
default=gr_request_queue,
|
||||
|
|
|
|||
|
|
@ -14,6 +14,6 @@ class TextCompletionImpl:
|
|||
self.context = context
|
||||
def invoke(self, **arguments):
|
||||
return self.context.prompt.request(
|
||||
"question", { "question": arguments.get("computation") }
|
||||
"question", { "question": arguments.get("question") }
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ class TrustGraph:
|
|||
|
||||
def __init__(
|
||||
self, hosts=None,
|
||||
keyspace="trustgraph", table="default",
|
||||
keyspace="trustgraph", table="default", username=None, password=None
|
||||
):
|
||||
|
||||
if hosts is None:
|
||||
|
|
@ -14,8 +14,13 @@ class TrustGraph:
|
|||
|
||||
self.keyspace = keyspace
|
||||
self.table = table
|
||||
self.username = username
|
||||
|
||||
self.cluster = Cluster(hosts)
|
||||
if username and password:
|
||||
auth_provider = PlainTextAuthProvider(username=username, password=password)
|
||||
self.cluster = Cluster(hosts, auth_provider=auth_provider)
|
||||
else:
|
||||
self.cluster = Cluster(hosts)
|
||||
self.session = self.cluster.connect()
|
||||
|
||||
self.init()
|
||||
|
|
|
|||
|
|
@ -16,6 +16,44 @@ from . schema import document_embeddings_response_queue
|
|||
LABEL="http://www.w3.org/2000/01/rdf-schema#label"
|
||||
DEFINITION="http://www.w3.org/2004/02/skos/core#definition"
|
||||
|
||||
class Query:
|
||||
|
||||
def __init__(self, rag, user, collection, verbose):
|
||||
self.rag = rag
|
||||
self.user = user
|
||||
self.collection = collection
|
||||
self.verbose = verbose
|
||||
|
||||
def get_vector(self, query):
|
||||
|
||||
if self.verbose:
|
||||
print("Compute embeddings...", flush=True)
|
||||
|
||||
qembeds = self.rag.embeddings.request(query)
|
||||
|
||||
if self.verbose:
|
||||
print("Done.", flush=True)
|
||||
|
||||
return qembeds
|
||||
|
||||
def get_docs(self, query):
|
||||
|
||||
vectors = self.get_vector(query)
|
||||
|
||||
if self.verbose:
|
||||
print("Get entities...", flush=True)
|
||||
|
||||
docs = self.rag.de_client.request(
|
||||
vectors, limit=self.rag.doc_limit
|
||||
)
|
||||
|
||||
if self.verbose:
|
||||
print("Docs:", flush=True)
|
||||
for doc in docs:
|
||||
print(doc, flush=True)
|
||||
|
||||
return docs
|
||||
|
||||
class DocumentRag:
|
||||
|
||||
def __init__(
|
||||
|
|
@ -56,7 +94,7 @@ class DocumentRag:
|
|||
print("Initialising...", flush=True)
|
||||
|
||||
# FIXME: Configurable
|
||||
self.entity_limit = 20
|
||||
self.doc_limit = 20
|
||||
|
||||
self.de_client = DocumentEmbeddingsClient(
|
||||
pulsar_host=pulsar_host,
|
||||
|
|
@ -85,42 +123,16 @@ class DocumentRag:
|
|||
if self.verbose:
|
||||
print("Initialised", flush=True)
|
||||
|
||||
def get_vector(self, query):
|
||||
|
||||
if self.verbose:
|
||||
print("Compute embeddings...", flush=True)
|
||||
|
||||
qembeds = self.embeddings.request(query)
|
||||
|
||||
if self.verbose:
|
||||
print("Done.", flush=True)
|
||||
|
||||
return qembeds
|
||||
|
||||
def get_docs(self, query):
|
||||
|
||||
vectors = self.get_vector(query)
|
||||
|
||||
if self.verbose:
|
||||
print("Get entities...", flush=True)
|
||||
|
||||
docs = self.de_client.request(
|
||||
vectors, self.entity_limit
|
||||
)
|
||||
|
||||
if self.verbose:
|
||||
print("Docs:", flush=True)
|
||||
for doc in docs:
|
||||
print(doc, flush=True)
|
||||
|
||||
return docs
|
||||
|
||||
def query(self, query):
|
||||
def query(self, query, user="trustgraph", collection="default"):
|
||||
|
||||
if self.verbose:
|
||||
print("Construct prompt...", flush=True)
|
||||
|
||||
docs = self.get_docs(query)
|
||||
q = Query(
|
||||
rag=self, user=user, collection=collection, verbose=self.verbose
|
||||
)
|
||||
|
||||
docs = q.get_docs(query)
|
||||
|
||||
if self.verbose:
|
||||
print("Invoke LLM...", flush=True)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,3 @@
|
|||
|
||||
from . embeddings import *
|
||||
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
|
||||
from . vectorize import run
|
||||
from . embeddings import run
|
||||
|
||||
if __name__ == '__main__':
|
||||
run()
|
||||
|
|
@ -1,11 +1,13 @@
|
|||
|
||||
"""
|
||||
Vectorizer, calls the embeddings service to get embeddings for a chunk.
|
||||
Input is text chunk, output is chunk and vectors.
|
||||
Document embeddings, calls the embeddings service to get embeddings for a
|
||||
chunk of text. Input is chunk of text plus metadata.
|
||||
Output is chunk plus embedding.
|
||||
"""
|
||||
|
||||
from ... schema import Chunk, ChunkEmbeddings
|
||||
from ... schema import chunk_ingest_queue, chunk_embeddings_ingest_queue
|
||||
from ... schema import Chunk, ChunkEmbeddings, DocumentEmbeddings
|
||||
from ... schema import chunk_ingest_queue
|
||||
from ... schema import document_embeddings_store_queue
|
||||
from ... schema import embeddings_request_queue, embeddings_response_queue
|
||||
from ... clients.embeddings_client import EmbeddingsClient
|
||||
from ... log_level import LogLevel
|
||||
|
|
@ -14,7 +16,7 @@ from ... base import ConsumerProducer
|
|||
module = ".".join(__name__.split(".")[1:-1])
|
||||
|
||||
default_input_queue = chunk_ingest_queue
|
||||
default_output_queue = chunk_embeddings_ingest_queue
|
||||
default_output_queue = document_embeddings_store_queue
|
||||
default_subscriber = module
|
||||
|
||||
class Processor(ConsumerProducer):
|
||||
|
|
@ -39,7 +41,7 @@ class Processor(ConsumerProducer):
|
|||
"embeddings_response_queue": emb_response_queue,
|
||||
"subscriber": subscriber,
|
||||
"input_schema": Chunk,
|
||||
"output_schema": ChunkEmbeddings,
|
||||
"output_schema": DocumentEmbeddings,
|
||||
}
|
||||
)
|
||||
|
||||
|
|
@ -51,31 +53,35 @@ class Processor(ConsumerProducer):
|
|||
subscriber=module + "-emb",
|
||||
)
|
||||
|
||||
def emit(self, metadata, chunk, vectors):
|
||||
|
||||
r = ChunkEmbeddings(metadata=metadata, chunk=chunk, vectors=vectors)
|
||||
self.producer.send(r)
|
||||
|
||||
def handle(self, msg):
|
||||
|
||||
v = msg.value()
|
||||
print(f"Indexing {v.metadata.id}...", flush=True)
|
||||
|
||||
chunk = v.chunk.decode("utf-8")
|
||||
|
||||
try:
|
||||
|
||||
vectors = self.embeddings.request(chunk)
|
||||
vectors = self.embeddings.request(v.chunk)
|
||||
|
||||
self.emit(
|
||||
embeds = [
|
||||
ChunkEmbeddings(
|
||||
chunk=v.chunk,
|
||||
vectors=vectors,
|
||||
)
|
||||
]
|
||||
|
||||
r = DocumentEmbeddings(
|
||||
metadata=v.metadata,
|
||||
chunk=chunk.encode("utf-8"),
|
||||
vectors=vectors
|
||||
chunks=embeds,
|
||||
)
|
||||
|
||||
self.producer.send(r)
|
||||
|
||||
except Exception as e:
|
||||
print("Exception:", e, flush=True)
|
||||
|
||||
# Retry
|
||||
raise e
|
||||
|
||||
print("Done.", flush=True)
|
||||
|
||||
@staticmethod
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
|
||||
from . processor import *
|
||||
|
||||
7
trustgraph-flow/trustgraph/embeddings/fastembed/__main__.py
Executable file
7
trustgraph-flow/trustgraph/embeddings/fastembed/__main__.py
Executable file
|
|
@ -0,0 +1,7 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
from . processor import run
|
||||
|
||||
if __name__ == '__main__':
|
||||
run()
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Add a link
Reference in a new issue