apunkt/trustgraph
1
0
Fork 0
mirror of https://github.com/trustgraph-ai/trustgraph.git synced 2026-04-25 00:16:23 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Add multi-arch (amd64/arm64) container builds and parallel CI

Browse source 
Pull requests back-ported to release/v2.2: 801, 802, 804, 805

Restructure container builds for multi-platform support, enabling
ARM-based deployments (e.g. Apple Silicon via Docker Desktop).

Makefile:
- Replace per-container named targets with pattern rules
  (container-%, manifest-%, platform-%-{amd64,arm64},
  combine-manifest-%)
- Add parallel CI targets: platform builds push per-arch images,
  combine-manifest creates and pushes the multi-arch manifest list
- Remove legacy cruft targets (update-dcs, update-templates)

CI (release.yaml):
- Split single deploy job into build-platform-image (16 parallel
  jobs: 8 containers x 2 platforms) and combine-manifests (8 jobs,
  metadata only)
- Use native ARM runners (ubuntu-24.04-arm)

Containerfile.hf:
- Downgrade to Python 3.12 (PyTorch lacks arm64 wheels for 3.13)
- Use standard PyTorch package instead of +cpu variant (no arm64 wheels
  on the cpu index)
This commit is contained in:
cybermaggedon 2026-04-13 20:51:09 +01:00 • committed by Cyber MacGeddon
parent dfb6d26a56
commit 03c7a7c5a8
3 changed files with 121 additions and 155 deletions
Download patch file Download diff file Expand all files Collapse all files

79
.github/workflows/release.yaml vendored
View file

@ -40,10 +40,9 @@ jobs:
- name: Publish release distributions to PyPI - name: Publish release distributions to PyPI
uses: pypa/gh-action-pypi-publish@release/v1 uses: pypa/gh-action-pypi-publish@release/v1
deploy-container-image: build-platform-image:
name: Release container images name: Build ${{ matrix.container }} (${{ matrix.platform }})
runs-on: ubuntu-24.04
permissions: permissions:
contents: write contents: write
id-token: write id-token: write
@ -52,14 +51,24 @@ jobs:
strategy: strategy:
matrix: matrix:
container: container:
- trustgraph-base - base
- trustgraph-flow - flow
- trustgraph-bedrock - bedrock
- trustgraph-vertexai - vertexai
- trustgraph-hf - hf
- trustgraph-ocr - ocr
- trustgraph-unstructured - unstructured
- trustgraph-mcp - mcp
platform:
- amd64
- arm64
include:
- platform: amd64
runner: ubuntu-24.04
- platform: arm64
runner: ubuntu-24.04-arm
runs-on: ${{ matrix.runner }}
steps: steps:
@ -76,12 +85,48 @@ jobs:
id: version id: version
run: echo VERSION=$(git describe --exact-match --tags | sed 's/^v//') >> $GITHUB_OUTPUT run: echo VERSION=$(git describe --exact-match --tags | sed 's/^v//') >> $GITHUB_OUTPUT
- name: Put version into package manifests - name: Build container
run: make update-package-versions VERSION=${{ steps.version.outputs.VERSION }} run: make platform-${{ matrix.container }}-${{ matrix.platform }} VERSION=${{ steps.version.outputs.VERSION }}
- name: Build container - ${{ matrix.container }} - name: Push container
run: make container-${{ matrix.container }} VERSION=${{ steps.version.outputs.VERSION }} run: make push-platform-${{ matrix.container }}-${{ matrix.platform }} VERSION=${{ steps.version.outputs.VERSION }}
- name: Push container - ${{ matrix.container }} combine-manifests:
run: make push-${{ matrix.container }} VERSION=${{ steps.version.outputs.VERSION }}
name: Combine manifest ${{ matrix.container }}
runs-on: ubuntu-24.04
needs: build-platform-image
permissions:
contents: write
id-token: write
environment:
name: release
strategy:
matrix:
container:
- base
- flow
- bedrock
- vertexai
- hf
- ocr
- unstructured
- mcp
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Docker Hub token
run: echo ${{ secrets.DOCKER_SECRET }} > docker-token.txt
- name: Authenticate with Docker hub
run: make docker-hub-login
- name: Get version
id: version
run: echo VERSION=$(git describe --exact-match --tags | sed 's/^v//') >> $GITHUB_OUTPUT
- name: Combine and push manifest
run: make combine-manifest-${{ matrix.container }} VERSION=${{ steps.version.outputs.VERSION }}

181
Makefile
View file

@ -52,51 +52,12 @@ update-package-versions:
echo __version__ = \"${VERSION}\" > trustgraph/trustgraph/trustgraph_version.py echo __version__ = \"${VERSION}\" > trustgraph/trustgraph/trustgraph_version.py
echo __version__ = \"${VERSION}\" > trustgraph-mcp/trustgraph/mcp_version.py echo __version__ = \"${VERSION}\" > trustgraph-mcp/trustgraph/mcp_version.py
FORCE: containers: container-base container-flow \
container-bedrock container-vertexai \
container-hf container-ocr \
container-unstructured container-mcp
containers: FORCE some-containers: container-base container-flow
${DOCKER} build -f containers/Containerfile.base \
-t ${CONTAINER_BASE}/trustgraph-base:${VERSION} .
${DOCKER} build -f containers/Containerfile.flow \
-t ${CONTAINER_BASE}/trustgraph-flow:${VERSION} .
${DOCKER} build -f containers/Containerfile.bedrock \
-t ${CONTAINER_BASE}/trustgraph-bedrock:${VERSION} .
${DOCKER} build -f containers/Containerfile.vertexai \
-t ${CONTAINER_BASE}/trustgraph-vertexai:${VERSION} .
${DOCKER} build -f containers/Containerfile.hf \
-t ${CONTAINER_BASE}/trustgraph-hf:${VERSION} .
${DOCKER} build -f containers/Containerfile.ocr \
-t ${CONTAINER_BASE}/trustgraph-ocr:${VERSION} .
${DOCKER} build -f containers/Containerfile.unstructured \
-t ${CONTAINER_BASE}/trustgraph-unstructured:${VERSION} .
${DOCKER} build -f containers/Containerfile.mcp \
-t ${CONTAINER_BASE}/trustgraph-mcp:${VERSION} .
some-containers:
${DOCKER} build -f containers/Containerfile.base \
-t ${CONTAINER_BASE}/trustgraph-base:${VERSION} .
${DOCKER} build -f containers/Containerfile.flow \
-t ${CONTAINER_BASE}/trustgraph-flow:${VERSION} .
# ${DOCKER} build -f containers/Containerfile.unstructured \
# -t ${CONTAINER_BASE}/trustgraph-unstructured:${VERSION} .
# ${DOCKER} build -f containers/Containerfile.vertexai \
# -t ${CONTAINER_BASE}/trustgraph-vertexai:${VERSION} .
# ${DOCKER} build -f containers/Containerfile.mcp \
# -t ${CONTAINER_BASE}/trustgraph-mcp:${VERSION} .
# ${DOCKER} build -f containers/Containerfile.vertexai \
# -t ${CONTAINER_BASE}/trustgraph-vertexai:${VERSION} .
# ${DOCKER} build -f containers/Containerfile.bedrock \
# -t ${CONTAINER_BASE}/trustgraph-bedrock:${VERSION} .
basic-containers: update-package-versions
${DOCKER} build -f containers/Containerfile.base \
-t ${CONTAINER_BASE}/trustgraph-base:${VERSION} .
${DOCKER} build -f containers/Containerfile.flow \
-t ${CONTAINER_BASE}/trustgraph-flow:${VERSION} .
container.ocr:
${DOCKER} build -f containers/Containerfile.ocr \
-t ${CONTAINER_BASE}/trustgraph-ocr:${VERSION} .
push: push:
${DOCKER} push ${CONTAINER_BASE}/trustgraph-base:${VERSION} ${DOCKER} push ${CONTAINER_BASE}/trustgraph-base:${VERSION}
@ -109,54 +70,60 @@ push:
${DOCKER} push ${CONTAINER_BASE}/trustgraph-mcp:${VERSION} ${DOCKER} push ${CONTAINER_BASE}/trustgraph-mcp:${VERSION}
# Individual container build targets # Individual container build targets
container-trustgraph-base: update-package-versions container-%: update-package-versions
${DOCKER} build -f containers/Containerfile.base -t ${CONTAINER_BASE}/trustgraph-base:${VERSION} . ${DOCKER} build \
-f containers/Containerfile.${@:container-%=%} \
-t ${CONTAINER_BASE}/trustgraph-${@:container-%=%}:${VERSION} .
container-trustgraph-flow: update-package-versions # Multi-arch: build both platforms sequentially into one manifest (local use)
${DOCKER} build -f containers/Containerfile.flow -t ${CONTAINER_BASE}/trustgraph-flow:${VERSION} . manifest-%: update-package-versions
-@${DOCKER} manifest rm \
${CONTAINER_BASE}/trustgraph-${@:manifest-%=%}:${VERSION}
${DOCKER} build --platform linux/amd64,linux/arm64 \
-f containers/Containerfile.${@:manifest-%=%} \
--manifest \
${CONTAINER_BASE}/trustgraph-${@:manifest-%=%}:${VERSION} .
container-trustgraph-bedrock: update-package-versions # Multi-arch: build a single platform image (for parallel CI)
${DOCKER} build -f containers/Containerfile.bedrock -t ${CONTAINER_BASE}/trustgraph-bedrock:${VERSION} . platform-%-amd64: update-package-versions
${DOCKER} build --platform linux/amd64 \
-f containers/Containerfile.${@:platform-%-amd64=%} \
-t ${CONTAINER_BASE}/trustgraph-${@:platform-%-amd64=%}:${VERSION}-amd64 .
container-trustgraph-vertexai: update-package-versions platform-%-arm64: update-package-versions
${DOCKER} build -f containers/Containerfile.vertexai -t ${CONTAINER_BASE}/trustgraph-vertexai:${VERSION} . ${DOCKER} build --platform linux/arm64 \
-f containers/Containerfile.${@:platform-%-arm64=%} \
-t ${CONTAINER_BASE}/trustgraph-${@:platform-%-arm64=%}:${VERSION}-arm64 .
container-trustgraph-hf: update-package-versions # Push a single platform image
${DOCKER} build -f containers/Containerfile.hf -t ${CONTAINER_BASE}/trustgraph-hf:${VERSION} . push-platform-%-amd64:
${DOCKER} push \
${CONTAINER_BASE}/trustgraph-${@:push-platform-%-amd64=%}:${VERSION}-amd64
container-trustgraph-ocr: update-package-versions push-platform-%-arm64:
${DOCKER} build -f containers/Containerfile.ocr -t ${CONTAINER_BASE}/trustgraph-ocr:${VERSION} . ${DOCKER} push \
${CONTAINER_BASE}/trustgraph-${@:push-platform-%-arm64=%}:${VERSION}-arm64
container-trustgraph-unstructured: update-package-versions # Combine per-platform images into a multi-arch manifest
${DOCKER} build -f containers/Containerfile.unstructured -t ${CONTAINER_BASE}/trustgraph-unstructured:${VERSION} . combine-manifest-%:
-@${DOCKER} manifest rm \
${CONTAINER_BASE}/trustgraph-${@:combine-manifest-%=%}:${VERSION}
${DOCKER} manifest create \
${CONTAINER_BASE}/trustgraph-${@:combine-manifest-%=%}:${VERSION} \
docker://${CONTAINER_BASE}/trustgraph-${@:combine-manifest-%=%}:${VERSION}-amd64 \
docker://${CONTAINER_BASE}/trustgraph-${@:combine-manifest-%=%}:${VERSION}-arm64
${DOCKER} manifest push \
${CONTAINER_BASE}/trustgraph-${@:combine-manifest-%=%}:${VERSION}
container-trustgraph-mcp: update-package-versions # Push a container
${DOCKER} build -f containers/Containerfile.mcp -t ${CONTAINER_BASE}/trustgraph-mcp:${VERSION} . push-container-%:
${DOCKER} push \
${CONTAINER_BASE}/trustgraph-${@:push-container-%=%}:${VERSION}
# Individual container push targets # Push a manifest (from local multi-arch build)
push-trustgraph-base: push-manifest-%:
${DOCKER} push ${CONTAINER_BASE}/trustgraph-base:${VERSION} ${DOCKER} manifest push \
${CONTAINER_BASE}/trustgraph-${@:push-manifest-%=%}:${VERSION}
push-trustgraph-flow:
${DOCKER} push ${CONTAINER_BASE}/trustgraph-flow:${VERSION}
push-trustgraph-bedrock:
${DOCKER} push ${CONTAINER_BASE}/trustgraph-bedrock:${VERSION}
push-trustgraph-vertexai:
${DOCKER} push ${CONTAINER_BASE}/trustgraph-vertexai:${VERSION}
push-trustgraph-hf:
${DOCKER} push ${CONTAINER_BASE}/trustgraph-hf:${VERSION}
push-trustgraph-ocr:
${DOCKER} push ${CONTAINER_BASE}/trustgraph-ocr:${VERSION}
push-trustgraph-unstructured:
${DOCKER} push ${CONTAINER_BASE}/trustgraph-unstructured:${VERSION}
push-trustgraph-mcp:
${DOCKER} push ${CONTAINER_BASE}/trustgraph-mcp:${VERSION}
clean: clean:
rm -rf wheels/ rm -rf wheels/
@ -164,52 +131,6 @@ clean:
set-version: set-version:
echo '"${VERSION}"' > templates/values/version.jsonnet echo '"${VERSION}"' > templates/values/version.jsonnet
TEMPLATES=azure bedrock claude cohere mix llamafile mistral ollama openai vertexai \
openai-neo4j storage
DCS=$(foreach template,${TEMPLATES},${template:%=tg-launch-%.yaml})
MODELS=azure bedrock claude cohere llamafile mistral ollama openai vertexai
GRAPHS=cassandra neo4j falkordb memgraph
# tg-launch-%.yaml: templates/%.jsonnet templates/components/version.jsonnet
# jsonnet -Jtemplates \
# -S ${@:tg-launch-%.yaml=templates/%.jsonnet} > $@
# VECTORDB=milvus
VECTORDB=qdrant
JSONNET_FLAGS=-J templates -J .
# Temporarily going back to how templates were built in 0.9 because this
# is going away in 0.11.
update-templates: update-dcs
JSON_TO_YAML=python -c 'import sys, yaml, json; j=json.loads(sys.stdin.read()); print(yaml.safe_dump(j))'
update-dcs: set-version
for graph in ${GRAPHS}; do \
cm=$${graph},pulsar,${VECTORDB},grafana; \
input=templates/opts-to-docker-compose.jsonnet; \
output=tg-storage-$${graph}.yaml; \
echo $${graph} '->' $${output}; \
jsonnet ${JSONNET_FLAGS} \
--ext-str options=$${cm} $${input} | \
${JSON_TO_YAML} > $${output}; \
done
for model in ${MODELS}; do \
for graph in ${GRAPHS}; do \
cm=$${graph},pulsar,${VECTORDB},embeddings-hf,graph-rag,grafana,trustgraph,$${model}; \
input=templates/opts-to-docker-compose.jsonnet; \
output=tg-launch-$${model}-$${graph}.yaml; \
echo $${model} + $${graph} '->' $${output}; \
jsonnet ${JSONNET_FLAGS} \
--ext-str options=$${cm} $${input} | \
${JSON_TO_YAML} > $${output}; \
done; \
done
docker-hub-login: docker-hub-login:
cat docker-token.txt | \ cat docker-token.txt | \
${DOCKER} login -u trustgraph --password-stdin registry-1.docker.io ${DOCKER} login -u trustgraph --password-stdin registry-1.docker.io

16
containers/Containerfile.hf
View file

@ -1,22 +1,22 @@
# ---------------------------------------------------------------------------- # Torch is stable and compiles for ARM64 and AMD64 on Python 3.12
# Build an AI container. This does the torch install which is huge, and I
# like to avoid re-doing this.
# ----------------------------------------------------------------------------
FROM docker.io/fedora:42 AS ai FROM docker.io/fedora:42 AS ai
ENV PIP_BREAK_SYSTEM_PACKAGES=1 ENV PIP_BREAK_SYSTEM_PACKAGES=1
RUN dnf install -y python3.13 && \ RUN dnf install -y python3.12 && \
alternatives --install /usr/bin/python python /usr/bin/python3.13 1 && \ alternatives --install /usr/bin/python python /usr/bin/python3.12 1 && \
python -m ensurepip --upgrade && \ python -m ensurepip --upgrade && \
pip3 install --no-cache-dir build wheel aiohttp && \ pip3 install --no-cache-dir build wheel aiohttp && \
pip3 install --no-cache-dir pulsar-client==3.7.0 && \ pip3 install --no-cache-dir pulsar-client==3.7.0 && \
dnf clean all dnf clean all
RUN pip3 install torch==2.5.1+cpu \ # This won't work on ARM
--index-url https://download.pytorch.org/whl/cpu #RUN pip3 install torch==2.5.1+cpu \
# --index-url https://download.pytorch.org/whl/cpu
RUN pip3 install torch
RUN pip3 install --no-cache-dir \ RUN pip3 install --no-cache-dir \
langchain==0.3.25 langchain-core==0.3.60 langchain-huggingface==0.2.0 \ langchain==0.3.25 langchain-core==0.3.60 langchain-huggingface==0.2.0 \