mirror of
https://github.com/trustgraph-ai/trustgraph.git
synced 2026-04-25 00:16:23 +02:00
Add multi-arch (amd64/arm64) container builds and parallel CI (#807)
Pull requests back-ported to release/v2.2: 801, 802, 804, 805
Restructure container builds for multi-platform support, enabling
ARM-based deployments (e.g. Apple Silicon via Docker Desktop).
Makefile:
- Replace per-container named targets with pattern rules
(container-%, manifest-%, platform-%-{amd64,arm64},
combine-manifest-%)
- Add parallel CI targets: platform builds push per-arch images,
combine-manifest creates and pushes the multi-arch manifest list
- Remove legacy cruft targets (update-dcs, update-templates)
CI (release.yaml):
- Split single deploy job into build-platform-image (16 parallel
jobs: 8 containers x 2 platforms) and combine-manifests (8 jobs,
metadata only)
- Use native ARM runners (ubuntu-24.04-arm)
Containerfile.hf:
- Downgrade to Python 3.12 (PyTorch lacks arm64 wheels for 3.13)
- Use standard PyTorch package instead of +cpu variant (no arm64 wheels
on the cpu index)
This commit is contained in:
cybermaggedon
GitHub
parent
dfb6d26a56
commit
b2e69cfa48
3 changed files with 121 additions and 155 deletions
79
.github/workflows/release.yaml
vendored
79
.github/workflows/release.yaml
vendored
|
|
@ -40,10 +40,9 @@ jobs:
|
||||||
- name: Publish release distributions to PyPI
|
- name: Publish release distributions to PyPI
|
||||||
uses: pypa/gh-action-pypi-publish@release/v1
|
uses: pypa/gh-action-pypi-publish@release/v1
|
||||||
|
|
||||||
deploy-container-image:
|
build-platform-image:
|
||||||
|
|
||||||
name: Release container images
|
name: Build ${{ matrix.container }} (${{ matrix.platform }})
|
||||||
runs-on: ubuntu-24.04
|
|
||||||
permissions:
|
permissions:
|
||||||
contents: write
|
contents: write
|
||||||
id-token: write
|
id-token: write
|
||||||
|
|
@ -52,14 +51,24 @@ jobs:
|
||||||
strategy:
|
strategy:
|
||||||
matrix:
|
matrix:
|
||||||
container:
|
container:
|
||||||
- trustgraph-base
|
- base
|
||||||
- trustgraph-flow
|
- flow
|
||||||
- trustgraph-bedrock
|
- bedrock
|
||||||
- trustgraph-vertexai
|
- vertexai
|
||||||
- trustgraph-hf
|
- hf
|
||||||
- trustgraph-ocr
|
- ocr
|
||||||
- trustgraph-unstructured
|
- unstructured
|
||||||
- trustgraph-mcp
|
- mcp
|
||||||
|
platform:
|
||||||
|
- amd64
|
||||||
|
- arm64
|
||||||
|
include:
|
||||||
|
- platform: amd64
|
||||||
|
runner: ubuntu-24.04
|
||||||
|
- platform: arm64
|
||||||
|
runner: ubuntu-24.04-arm
|
||||||
|
|
||||||
|
runs-on: ${{ matrix.runner }}
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
|
|
||||||
|
|
@ -76,12 +85,48 @@ jobs:
|
||||||
id: version
|
id: version
|
||||||
run: echo VERSION=$(git describe --exact-match --tags | sed 's/^v//') >> $GITHUB_OUTPUT
|
run: echo VERSION=$(git describe --exact-match --tags | sed 's/^v//') >> $GITHUB_OUTPUT
|
||||||
|
|
||||||
- name: Put version into package manifests
|
- name: Build container
|
||||||
run: make update-package-versions VERSION=${{ steps.version.outputs.VERSION }}
|
run: make platform-${{ matrix.container }}-${{ matrix.platform }} VERSION=${{ steps.version.outputs.VERSION }}
|
||||||
|
|
||||||
- name: Build container - ${{ matrix.container }}
|
- name: Push container
|
||||||
run: make container-${{ matrix.container }} VERSION=${{ steps.version.outputs.VERSION }}
|
run: make push-platform-${{ matrix.container }}-${{ matrix.platform }} VERSION=${{ steps.version.outputs.VERSION }}
|
||||||
|
|
||||||
- name: Push container - ${{ matrix.container }}
|
combine-manifests:
|
||||||
run: make push-${{ matrix.container }} VERSION=${{ steps.version.outputs.VERSION }}
|
|
||||||
|
|
||||||
|
name: Combine manifest ${{ matrix.container }}
|
||||||
|
runs-on: ubuntu-24.04
|
||||||
|
needs: build-platform-image
|
||||||
|
permissions:
|
||||||
|
contents: write
|
||||||
|
id-token: write
|
||||||
|
environment:
|
||||||
|
name: release
|
||||||
|
strategy:
|
||||||
|
matrix:
|
||||||
|
container:
|
||||||
|
- base
|
||||||
|
- flow
|
||||||
|
- bedrock
|
||||||
|
- vertexai
|
||||||
|
- hf
|
||||||
|
- ocr
|
||||||
|
- unstructured
|
||||||
|
- mcp
|
||||||
|
|
||||||
|
steps:
|
||||||
|
|
||||||
|
- name: Checkout
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Docker Hub token
|
||||||
|
run: echo ${{ secrets.DOCKER_SECRET }} > docker-token.txt
|
||||||
|
|
||||||
|
- name: Authenticate with Docker hub
|
||||||
|
run: make docker-hub-login
|
||||||
|
|
||||||
|
- name: Get version
|
||||||
|
id: version
|
||||||
|
run: echo VERSION=$(git describe --exact-match --tags | sed 's/^v//') >> $GITHUB_OUTPUT
|
||||||
|
|
||||||
|
- name: Combine and push manifest
|
||||||
|
run: make combine-manifest-${{ matrix.container }} VERSION=${{ steps.version.outputs.VERSION }}
|
||||||
|
|
|
||||||
181
Makefile
181
Makefile
|
|
@ -52,51 +52,12 @@ update-package-versions:
|
||||||
echo __version__ = \"${VERSION}\" > trustgraph/trustgraph/trustgraph_version.py
|
echo __version__ = \"${VERSION}\" > trustgraph/trustgraph/trustgraph_version.py
|
||||||
echo __version__ = \"${VERSION}\" > trustgraph-mcp/trustgraph/mcp_version.py
|
echo __version__ = \"${VERSION}\" > trustgraph-mcp/trustgraph/mcp_version.py
|
||||||
|
|
||||||
FORCE:
|
containers: container-base container-flow \
|
||||||
|
container-bedrock container-vertexai \
|
||||||
|
container-hf container-ocr \
|
||||||
|
container-unstructured container-mcp
|
||||||
|
|
||||||
containers: FORCE
|
some-containers: container-base container-flow
|
||||||
${DOCKER} build -f containers/Containerfile.base \
|
|
||||||
-t ${CONTAINER_BASE}/trustgraph-base:${VERSION} .
|
|
||||||
${DOCKER} build -f containers/Containerfile.flow \
|
|
||||||
-t ${CONTAINER_BASE}/trustgraph-flow:${VERSION} .
|
|
||||||
${DOCKER} build -f containers/Containerfile.bedrock \
|
|
||||||
-t ${CONTAINER_BASE}/trustgraph-bedrock:${VERSION} .
|
|
||||||
${DOCKER} build -f containers/Containerfile.vertexai \
|
|
||||||
-t ${CONTAINER_BASE}/trustgraph-vertexai:${VERSION} .
|
|
||||||
${DOCKER} build -f containers/Containerfile.hf \
|
|
||||||
-t ${CONTAINER_BASE}/trustgraph-hf:${VERSION} .
|
|
||||||
${DOCKER} build -f containers/Containerfile.ocr \
|
|
||||||
-t ${CONTAINER_BASE}/trustgraph-ocr:${VERSION} .
|
|
||||||
${DOCKER} build -f containers/Containerfile.unstructured \
|
|
||||||
-t ${CONTAINER_BASE}/trustgraph-unstructured:${VERSION} .
|
|
||||||
${DOCKER} build -f containers/Containerfile.mcp \
|
|
||||||
-t ${CONTAINER_BASE}/trustgraph-mcp:${VERSION} .
|
|
||||||
|
|
||||||
some-containers:
|
|
||||||
${DOCKER} build -f containers/Containerfile.base \
|
|
||||||
-t ${CONTAINER_BASE}/trustgraph-base:${VERSION} .
|
|
||||||
${DOCKER} build -f containers/Containerfile.flow \
|
|
||||||
-t ${CONTAINER_BASE}/trustgraph-flow:${VERSION} .
|
|
||||||
# ${DOCKER} build -f containers/Containerfile.unstructured \
|
|
||||||
# -t ${CONTAINER_BASE}/trustgraph-unstructured:${VERSION} .
|
|
||||||
# ${DOCKER} build -f containers/Containerfile.vertexai \
|
|
||||||
# -t ${CONTAINER_BASE}/trustgraph-vertexai:${VERSION} .
|
|
||||||
# ${DOCKER} build -f containers/Containerfile.mcp \
|
|
||||||
# -t ${CONTAINER_BASE}/trustgraph-mcp:${VERSION} .
|
|
||||||
# ${DOCKER} build -f containers/Containerfile.vertexai \
|
|
||||||
# -t ${CONTAINER_BASE}/trustgraph-vertexai:${VERSION} .
|
|
||||||
# ${DOCKER} build -f containers/Containerfile.bedrock \
|
|
||||||
# -t ${CONTAINER_BASE}/trustgraph-bedrock:${VERSION} .
|
|
||||||
|
|
||||||
basic-containers: update-package-versions
|
|
||||||
${DOCKER} build -f containers/Containerfile.base \
|
|
||||||
-t ${CONTAINER_BASE}/trustgraph-base:${VERSION} .
|
|
||||||
${DOCKER} build -f containers/Containerfile.flow \
|
|
||||||
-t ${CONTAINER_BASE}/trustgraph-flow:${VERSION} .
|
|
||||||
|
|
||||||
container.ocr:
|
|
||||||
${DOCKER} build -f containers/Containerfile.ocr \
|
|
||||||
-t ${CONTAINER_BASE}/trustgraph-ocr:${VERSION} .
|
|
||||||
|
|
||||||
push:
|
push:
|
||||||
${DOCKER} push ${CONTAINER_BASE}/trustgraph-base:${VERSION}
|
${DOCKER} push ${CONTAINER_BASE}/trustgraph-base:${VERSION}
|
||||||
|
|
@ -109,54 +70,60 @@ push:
|
||||||
${DOCKER} push ${CONTAINER_BASE}/trustgraph-mcp:${VERSION}
|
${DOCKER} push ${CONTAINER_BASE}/trustgraph-mcp:${VERSION}
|
||||||
|
|
||||||
# Individual container build targets
|
# Individual container build targets
|
||||||
container-trustgraph-base: update-package-versions
|
container-%: update-package-versions
|
||||||
${DOCKER} build -f containers/Containerfile.base -t ${CONTAINER_BASE}/trustgraph-base:${VERSION} .
|
${DOCKER} build \
|
||||||
|
-f containers/Containerfile.${@:container-%=%} \
|
||||||
|
-t ${CONTAINER_BASE}/trustgraph-${@:container-%=%}:${VERSION} .
|
||||||
|
|
||||||
container-trustgraph-flow: update-package-versions
|
# Multi-arch: build both platforms sequentially into one manifest (local use)
|
||||||
${DOCKER} build -f containers/Containerfile.flow -t ${CONTAINER_BASE}/trustgraph-flow:${VERSION} .
|
manifest-%: update-package-versions
|
||||||
|
-@${DOCKER} manifest rm \
|
||||||
|
${CONTAINER_BASE}/trustgraph-${@:manifest-%=%}:${VERSION}
|
||||||
|
${DOCKER} build --platform linux/amd64,linux/arm64 \
|
||||||
|
-f containers/Containerfile.${@:manifest-%=%} \
|
||||||
|
--manifest \
|
||||||
|
${CONTAINER_BASE}/trustgraph-${@:manifest-%=%}:${VERSION} .
|
||||||
|
|
||||||
container-trustgraph-bedrock: update-package-versions
|
# Multi-arch: build a single platform image (for parallel CI)
|
||||||
${DOCKER} build -f containers/Containerfile.bedrock -t ${CONTAINER_BASE}/trustgraph-bedrock:${VERSION} .
|
platform-%-amd64: update-package-versions
|
||||||
|
${DOCKER} build --platform linux/amd64 \
|
||||||
|
-f containers/Containerfile.${@:platform-%-amd64=%} \
|
||||||
|
-t ${CONTAINER_BASE}/trustgraph-${@:platform-%-amd64=%}:${VERSION}-amd64 .
|
||||||
|
|
||||||
container-trustgraph-vertexai: update-package-versions
|
platform-%-arm64: update-package-versions
|
||||||
${DOCKER} build -f containers/Containerfile.vertexai -t ${CONTAINER_BASE}/trustgraph-vertexai:${VERSION} .
|
${DOCKER} build --platform linux/arm64 \
|
||||||
|
-f containers/Containerfile.${@:platform-%-arm64=%} \
|
||||||
|
-t ${CONTAINER_BASE}/trustgraph-${@:platform-%-arm64=%}:${VERSION}-arm64 .
|
||||||
|
|
||||||
container-trustgraph-hf: update-package-versions
|
# Push a single platform image
|
||||||
${DOCKER} build -f containers/Containerfile.hf -t ${CONTAINER_BASE}/trustgraph-hf:${VERSION} .
|
push-platform-%-amd64:
|
||||||
|
${DOCKER} push \
|
||||||
|
${CONTAINER_BASE}/trustgraph-${@:push-platform-%-amd64=%}:${VERSION}-amd64
|
||||||
|
|
||||||
container-trustgraph-ocr: update-package-versions
|
push-platform-%-arm64:
|
||||||
${DOCKER} build -f containers/Containerfile.ocr -t ${CONTAINER_BASE}/trustgraph-ocr:${VERSION} .
|
${DOCKER} push \
|
||||||
|
${CONTAINER_BASE}/trustgraph-${@:push-platform-%-arm64=%}:${VERSION}-arm64
|
||||||
|
|
||||||
container-trustgraph-unstructured: update-package-versions
|
# Combine per-platform images into a multi-arch manifest
|
||||||
${DOCKER} build -f containers/Containerfile.unstructured -t ${CONTAINER_BASE}/trustgraph-unstructured:${VERSION} .
|
combine-manifest-%:
|
||||||
|
-@${DOCKER} manifest rm \
|
||||||
|
${CONTAINER_BASE}/trustgraph-${@:combine-manifest-%=%}:${VERSION}
|
||||||
|
${DOCKER} manifest create \
|
||||||
|
${CONTAINER_BASE}/trustgraph-${@:combine-manifest-%=%}:${VERSION} \
|
||||||
|
docker://${CONTAINER_BASE}/trustgraph-${@:combine-manifest-%=%}:${VERSION}-amd64 \
|
||||||
|
docker://${CONTAINER_BASE}/trustgraph-${@:combine-manifest-%=%}:${VERSION}-arm64
|
||||||
|
${DOCKER} manifest push \
|
||||||
|
${CONTAINER_BASE}/trustgraph-${@:combine-manifest-%=%}:${VERSION}
|
||||||
|
|
||||||
container-trustgraph-mcp: update-package-versions
|
# Push a container
|
||||||
${DOCKER} build -f containers/Containerfile.mcp -t ${CONTAINER_BASE}/trustgraph-mcp:${VERSION} .
|
push-container-%:
|
||||||
|
${DOCKER} push \
|
||||||
|
${CONTAINER_BASE}/trustgraph-${@:push-container-%=%}:${VERSION}
|
||||||
|
|
||||||
# Individual container push targets
|
# Push a manifest (from local multi-arch build)
|
||||||
push-trustgraph-base:
|
push-manifest-%:
|
||||||
${DOCKER} push ${CONTAINER_BASE}/trustgraph-base:${VERSION}
|
${DOCKER} manifest push \
|
||||||
|
${CONTAINER_BASE}/trustgraph-${@:push-manifest-%=%}:${VERSION}
|
||||||
push-trustgraph-flow:
|
|
||||||
${DOCKER} push ${CONTAINER_BASE}/trustgraph-flow:${VERSION}
|
|
||||||
|
|
||||||
push-trustgraph-bedrock:
|
|
||||||
${DOCKER} push ${CONTAINER_BASE}/trustgraph-bedrock:${VERSION}
|
|
||||||
|
|
||||||
push-trustgraph-vertexai:
|
|
||||||
${DOCKER} push ${CONTAINER_BASE}/trustgraph-vertexai:${VERSION}
|
|
||||||
|
|
||||||
push-trustgraph-hf:
|
|
||||||
${DOCKER} push ${CONTAINER_BASE}/trustgraph-hf:${VERSION}
|
|
||||||
|
|
||||||
push-trustgraph-ocr:
|
|
||||||
${DOCKER} push ${CONTAINER_BASE}/trustgraph-ocr:${VERSION}
|
|
||||||
|
|
||||||
push-trustgraph-unstructured:
|
|
||||||
${DOCKER} push ${CONTAINER_BASE}/trustgraph-unstructured:${VERSION}
|
|
||||||
|
|
||||||
push-trustgraph-mcp:
|
|
||||||
${DOCKER} push ${CONTAINER_BASE}/trustgraph-mcp:${VERSION}
|
|
||||||
|
|
||||||
clean:
|
clean:
|
||||||
rm -rf wheels/
|
rm -rf wheels/
|
||||||
|
|
@ -164,52 +131,6 @@ clean:
|
||||||
set-version:
|
set-version:
|
||||||
echo '"${VERSION}"' > templates/values/version.jsonnet
|
echo '"${VERSION}"' > templates/values/version.jsonnet
|
||||||
|
|
||||||
TEMPLATES=azure bedrock claude cohere mix llamafile mistral ollama openai vertexai \
|
|
||||||
openai-neo4j storage
|
|
||||||
|
|
||||||
DCS=$(foreach template,${TEMPLATES},${template:%=tg-launch-%.yaml})
|
|
||||||
|
|
||||||
MODELS=azure bedrock claude cohere llamafile mistral ollama openai vertexai
|
|
||||||
GRAPHS=cassandra neo4j falkordb memgraph
|
|
||||||
|
|
||||||
# tg-launch-%.yaml: templates/%.jsonnet templates/components/version.jsonnet
|
|
||||||
# jsonnet -Jtemplates \
|
|
||||||
# -S ${@:tg-launch-%.yaml=templates/%.jsonnet} > $@
|
|
||||||
|
|
||||||
# VECTORDB=milvus
|
|
||||||
VECTORDB=qdrant
|
|
||||||
|
|
||||||
JSONNET_FLAGS=-J templates -J .
|
|
||||||
|
|
||||||
# Temporarily going back to how templates were built in 0.9 because this
|
|
||||||
# is going away in 0.11.
|
|
||||||
|
|
||||||
update-templates: update-dcs
|
|
||||||
|
|
||||||
JSON_TO_YAML=python -c 'import sys, yaml, json; j=json.loads(sys.stdin.read()); print(yaml.safe_dump(j))'
|
|
||||||
|
|
||||||
update-dcs: set-version
|
|
||||||
for graph in ${GRAPHS}; do \
|
|
||||||
cm=$${graph},pulsar,${VECTORDB},grafana; \
|
|
||||||
input=templates/opts-to-docker-compose.jsonnet; \
|
|
||||||
output=tg-storage-$${graph}.yaml; \
|
|
||||||
echo $${graph} '->' $${output}; \
|
|
||||||
jsonnet ${JSONNET_FLAGS} \
|
|
||||||
--ext-str options=$${cm} $${input} | \
|
|
||||||
${JSON_TO_YAML} > $${output}; \
|
|
||||||
done
|
|
||||||
for model in ${MODELS}; do \
|
|
||||||
for graph in ${GRAPHS}; do \
|
|
||||||
cm=$${graph},pulsar,${VECTORDB},embeddings-hf,graph-rag,grafana,trustgraph,$${model}; \
|
|
||||||
input=templates/opts-to-docker-compose.jsonnet; \
|
|
||||||
output=tg-launch-$${model}-$${graph}.yaml; \
|
|
||||||
echo $${model} + $${graph} '->' $${output}; \
|
|
||||||
jsonnet ${JSONNET_FLAGS} \
|
|
||||||
--ext-str options=$${cm} $${input} | \
|
|
||||||
${JSON_TO_YAML} > $${output}; \
|
|
||||||
done; \
|
|
||||||
done
|
|
||||||
|
|
||||||
docker-hub-login:
|
docker-hub-login:
|
||||||
cat docker-token.txt | \
|
cat docker-token.txt | \
|
||||||
${DOCKER} login -u trustgraph --password-stdin registry-1.docker.io
|
${DOCKER} login -u trustgraph --password-stdin registry-1.docker.io
|
||||||
|
|
|
||||||
|
|
@ -1,22 +1,22 @@
|
||||||
|
|
||||||
# ----------------------------------------------------------------------------
|
# Torch is stable and compiles for ARM64 and AMD64 on Python 3.12
|
||||||
# Build an AI container. This does the torch install which is huge, and I
|
|
||||||
# like to avoid re-doing this.
|
|
||||||
# ----------------------------------------------------------------------------
|
|
||||||
|
|
||||||
FROM docker.io/fedora:42 AS ai
|
FROM docker.io/fedora:42 AS ai
|
||||||
|
|
||||||
ENV PIP_BREAK_SYSTEM_PACKAGES=1
|
ENV PIP_BREAK_SYSTEM_PACKAGES=1
|
||||||
|
|
||||||
RUN dnf install -y python3.13 && \
|
RUN dnf install -y python3.12 && \
|
||||||
alternatives --install /usr/bin/python python /usr/bin/python3.13 1 && \
|
alternatives --install /usr/bin/python python /usr/bin/python3.12 1 && \
|
||||||
python -m ensurepip --upgrade && \
|
python -m ensurepip --upgrade && \
|
||||||
pip3 install --no-cache-dir build wheel aiohttp && \
|
pip3 install --no-cache-dir build wheel aiohttp && \
|
||||||
pip3 install --no-cache-dir pulsar-client==3.7.0 && \
|
pip3 install --no-cache-dir pulsar-client==3.7.0 && \
|
||||||
dnf clean all
|
dnf clean all
|
||||||
|
|
||||||
RUN pip3 install torch==2.5.1+cpu \
|
# This won't work on ARM
|
||||||
--index-url https://download.pytorch.org/whl/cpu
|
#RUN pip3 install torch==2.5.1+cpu \
|
||||||
|
# --index-url https://download.pytorch.org/whl/cpu
|
||||||
|
|
||||||
|
RUN pip3 install torch
|
||||||
|
|
||||||
RUN pip3 install --no-cache-dir \
|
RUN pip3 install --no-cache-dir \
|
||||||
langchain==0.3.25 langchain-core==0.3.60 langchain-huggingface==0.2.0 \
|
langchain==0.3.25 langchain-core==0.3.60 langchain-huggingface==0.2.0 \
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue