mirror of
https://github.com/trustgraph-ai/trustgraph.git
synced 2026-04-24 16:06:22 +02:00
Add multi-arch (amd64/arm64) container builds and parallel CI
Pull requests back-ported to release/v2.2: 801, 802, 804, 805
Restructure container builds for multi-platform support, enabling
ARM-based deployments (e.g. Apple Silicon via Docker Desktop).
Makefile:
- Replace per-container named targets with pattern rules
(container-%, manifest-%, platform-%-{amd64,arm64},
combine-manifest-%)
- Add parallel CI targets: platform builds push per-arch images,
combine-manifest creates and pushes the multi-arch manifest list
- Remove legacy cruft targets (update-dcs, update-templates)
CI (release.yaml):
- Split single deploy job into build-platform-image (16 parallel
jobs: 8 containers x 2 platforms) and combine-manifests (8 jobs,
metadata only)
- Use native ARM runners (ubuntu-24.04-arm)
Containerfile.hf:
- Downgrade to Python 3.12 (PyTorch lacks arm64 wheels for 3.13)
- Use standard PyTorch package instead of +cpu variant (no arm64 wheels
on the cpu index)
This commit is contained in:
cybermaggedon
Cyber MacGeddon
parent
dfb6d26a56
commit
03c7a7c5a8
3 changed files with 121 additions and 155 deletions
79
.github/workflows/release.yaml
vendored
79
.github/workflows/release.yaml
vendored
|
|
@ -40,10 +40,9 @@ jobs:
|
|||
- name: Publish release distributions to PyPI
|
||||
uses: pypa/gh-action-pypi-publish@release/v1
|
||||
|
||||
deploy-container-image:
|
||||
build-platform-image:
|
||||
|
||||
name: Release container images
|
||||
runs-on: ubuntu-24.04
|
||||
name: Build ${{ matrix.container }} (${{ matrix.platform }})
|
||||
permissions:
|
||||
contents: write
|
||||
id-token: write
|
||||
|
|
@ -52,14 +51,24 @@ jobs:
|
|||
strategy:
|
||||
matrix:
|
||||
container:
|
||||
- trustgraph-base
|
||||
- trustgraph-flow
|
||||
- trustgraph-bedrock
|
||||
- trustgraph-vertexai
|
||||
- trustgraph-hf
|
||||
- trustgraph-ocr
|
||||
- trustgraph-unstructured
|
||||
- trustgraph-mcp
|
||||
- base
|
||||
- flow
|
||||
- bedrock
|
||||
- vertexai
|
||||
- hf
|
||||
- ocr
|
||||
- unstructured
|
||||
- mcp
|
||||
platform:
|
||||
- amd64
|
||||
- arm64
|
||||
include:
|
||||
- platform: amd64
|
||||
runner: ubuntu-24.04
|
||||
- platform: arm64
|
||||
runner: ubuntu-24.04-arm
|
||||
|
||||
runs-on: ${{ matrix.runner }}
|
||||
|
||||
steps:
|
||||
|
||||
|
|
@ -76,12 +85,48 @@ jobs:
|
|||
id: version
|
||||
run: echo VERSION=$(git describe --exact-match --tags | sed 's/^v//') >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Put version into package manifests
|
||||
run: make update-package-versions VERSION=${{ steps.version.outputs.VERSION }}
|
||||
- name: Build container
|
||||
run: make platform-${{ matrix.container }}-${{ matrix.platform }} VERSION=${{ steps.version.outputs.VERSION }}
|
||||
|
||||
- name: Build container - ${{ matrix.container }}
|
||||
run: make container-${{ matrix.container }} VERSION=${{ steps.version.outputs.VERSION }}
|
||||
- name: Push container
|
||||
run: make push-platform-${{ matrix.container }}-${{ matrix.platform }} VERSION=${{ steps.version.outputs.VERSION }}
|
||||
|
||||
- name: Push container - ${{ matrix.container }}
|
||||
run: make push-${{ matrix.container }} VERSION=${{ steps.version.outputs.VERSION }}
|
||||
combine-manifests:
|
||||
|
||||
name: Combine manifest ${{ matrix.container }}
|
||||
runs-on: ubuntu-24.04
|
||||
needs: build-platform-image
|
||||
permissions:
|
||||
contents: write
|
||||
id-token: write
|
||||
environment:
|
||||
name: release
|
||||
strategy:
|
||||
matrix:
|
||||
container:
|
||||
- base
|
||||
- flow
|
||||
- bedrock
|
||||
- vertexai
|
||||
- hf
|
||||
- ocr
|
||||
- unstructured
|
||||
- mcp
|
||||
|
||||
steps:
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Docker Hub token
|
||||
run: echo ${{ secrets.DOCKER_SECRET }} > docker-token.txt
|
||||
|
||||
- name: Authenticate with Docker hub
|
||||
run: make docker-hub-login
|
||||
|
||||
- name: Get version
|
||||
id: version
|
||||
run: echo VERSION=$(git describe --exact-match --tags | sed 's/^v//') >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Combine and push manifest
|
||||
run: make combine-manifest-${{ matrix.container }} VERSION=${{ steps.version.outputs.VERSION }}
|
||||
|
|
|
|||
181
Makefile
181
Makefile
|
|
@ -52,51 +52,12 @@ update-package-versions:
|
|||
echo __version__ = \"${VERSION}\" > trustgraph/trustgraph/trustgraph_version.py
|
||||
echo __version__ = \"${VERSION}\" > trustgraph-mcp/trustgraph/mcp_version.py
|
||||
|
||||
FORCE:
|
||||
containers: container-base container-flow \
|
||||
container-bedrock container-vertexai \
|
||||
container-hf container-ocr \
|
||||
container-unstructured container-mcp
|
||||
|
||||
containers: FORCE
|
||||
${DOCKER} build -f containers/Containerfile.base \
|
||||
-t ${CONTAINER_BASE}/trustgraph-base:${VERSION} .
|
||||
${DOCKER} build -f containers/Containerfile.flow \
|
||||
-t ${CONTAINER_BASE}/trustgraph-flow:${VERSION} .
|
||||
${DOCKER} build -f containers/Containerfile.bedrock \
|
||||
-t ${CONTAINER_BASE}/trustgraph-bedrock:${VERSION} .
|
||||
${DOCKER} build -f containers/Containerfile.vertexai \
|
||||
-t ${CONTAINER_BASE}/trustgraph-vertexai:${VERSION} .
|
||||
${DOCKER} build -f containers/Containerfile.hf \
|
||||
-t ${CONTAINER_BASE}/trustgraph-hf:${VERSION} .
|
||||
${DOCKER} build -f containers/Containerfile.ocr \
|
||||
-t ${CONTAINER_BASE}/trustgraph-ocr:${VERSION} .
|
||||
${DOCKER} build -f containers/Containerfile.unstructured \
|
||||
-t ${CONTAINER_BASE}/trustgraph-unstructured:${VERSION} .
|
||||
${DOCKER} build -f containers/Containerfile.mcp \
|
||||
-t ${CONTAINER_BASE}/trustgraph-mcp:${VERSION} .
|
||||
|
||||
some-containers:
|
||||
${DOCKER} build -f containers/Containerfile.base \
|
||||
-t ${CONTAINER_BASE}/trustgraph-base:${VERSION} .
|
||||
${DOCKER} build -f containers/Containerfile.flow \
|
||||
-t ${CONTAINER_BASE}/trustgraph-flow:${VERSION} .
|
||||
# ${DOCKER} build -f containers/Containerfile.unstructured \
|
||||
# -t ${CONTAINER_BASE}/trustgraph-unstructured:${VERSION} .
|
||||
# ${DOCKER} build -f containers/Containerfile.vertexai \
|
||||
# -t ${CONTAINER_BASE}/trustgraph-vertexai:${VERSION} .
|
||||
# ${DOCKER} build -f containers/Containerfile.mcp \
|
||||
# -t ${CONTAINER_BASE}/trustgraph-mcp:${VERSION} .
|
||||
# ${DOCKER} build -f containers/Containerfile.vertexai \
|
||||
# -t ${CONTAINER_BASE}/trustgraph-vertexai:${VERSION} .
|
||||
# ${DOCKER} build -f containers/Containerfile.bedrock \
|
||||
# -t ${CONTAINER_BASE}/trustgraph-bedrock:${VERSION} .
|
||||
|
||||
basic-containers: update-package-versions
|
||||
${DOCKER} build -f containers/Containerfile.base \
|
||||
-t ${CONTAINER_BASE}/trustgraph-base:${VERSION} .
|
||||
${DOCKER} build -f containers/Containerfile.flow \
|
||||
-t ${CONTAINER_BASE}/trustgraph-flow:${VERSION} .
|
||||
|
||||
container.ocr:
|
||||
${DOCKER} build -f containers/Containerfile.ocr \
|
||||
-t ${CONTAINER_BASE}/trustgraph-ocr:${VERSION} .
|
||||
some-containers: container-base container-flow
|
||||
|
||||
push:
|
||||
${DOCKER} push ${CONTAINER_BASE}/trustgraph-base:${VERSION}
|
||||
|
|
@ -109,54 +70,60 @@ push:
|
|||
${DOCKER} push ${CONTAINER_BASE}/trustgraph-mcp:${VERSION}
|
||||
|
||||
# Individual container build targets
|
||||
container-trustgraph-base: update-package-versions
|
||||
${DOCKER} build -f containers/Containerfile.base -t ${CONTAINER_BASE}/trustgraph-base:${VERSION} .
|
||||
container-%: update-package-versions
|
||||
${DOCKER} build \
|
||||
-f containers/Containerfile.${@:container-%=%} \
|
||||
-t ${CONTAINER_BASE}/trustgraph-${@:container-%=%}:${VERSION} .
|
||||
|
||||
container-trustgraph-flow: update-package-versions
|
||||
${DOCKER} build -f containers/Containerfile.flow -t ${CONTAINER_BASE}/trustgraph-flow:${VERSION} .
|
||||
# Multi-arch: build both platforms sequentially into one manifest (local use)
|
||||
manifest-%: update-package-versions
|
||||
-@${DOCKER} manifest rm \
|
||||
${CONTAINER_BASE}/trustgraph-${@:manifest-%=%}:${VERSION}
|
||||
${DOCKER} build --platform linux/amd64,linux/arm64 \
|
||||
-f containers/Containerfile.${@:manifest-%=%} \
|
||||
--manifest \
|
||||
${CONTAINER_BASE}/trustgraph-${@:manifest-%=%}:${VERSION} .
|
||||
|
||||
container-trustgraph-bedrock: update-package-versions
|
||||
${DOCKER} build -f containers/Containerfile.bedrock -t ${CONTAINER_BASE}/trustgraph-bedrock:${VERSION} .
|
||||
# Multi-arch: build a single platform image (for parallel CI)
|
||||
platform-%-amd64: update-package-versions
|
||||
${DOCKER} build --platform linux/amd64 \
|
||||
-f containers/Containerfile.${@:platform-%-amd64=%} \
|
||||
-t ${CONTAINER_BASE}/trustgraph-${@:platform-%-amd64=%}:${VERSION}-amd64 .
|
||||
|
||||
container-trustgraph-vertexai: update-package-versions
|
||||
${DOCKER} build -f containers/Containerfile.vertexai -t ${CONTAINER_BASE}/trustgraph-vertexai:${VERSION} .
|
||||
platform-%-arm64: update-package-versions
|
||||
${DOCKER} build --platform linux/arm64 \
|
||||
-f containers/Containerfile.${@:platform-%-arm64=%} \
|
||||
-t ${CONTAINER_BASE}/trustgraph-${@:platform-%-arm64=%}:${VERSION}-arm64 .
|
||||
|
||||
container-trustgraph-hf: update-package-versions
|
||||
${DOCKER} build -f containers/Containerfile.hf -t ${CONTAINER_BASE}/trustgraph-hf:${VERSION} .
|
||||
# Push a single platform image
|
||||
push-platform-%-amd64:
|
||||
${DOCKER} push \
|
||||
${CONTAINER_BASE}/trustgraph-${@:push-platform-%-amd64=%}:${VERSION}-amd64
|
||||
|
||||
container-trustgraph-ocr: update-package-versions
|
||||
${DOCKER} build -f containers/Containerfile.ocr -t ${CONTAINER_BASE}/trustgraph-ocr:${VERSION} .
|
||||
push-platform-%-arm64:
|
||||
${DOCKER} push \
|
||||
${CONTAINER_BASE}/trustgraph-${@:push-platform-%-arm64=%}:${VERSION}-arm64
|
||||
|
||||
container-trustgraph-unstructured: update-package-versions
|
||||
${DOCKER} build -f containers/Containerfile.unstructured -t ${CONTAINER_BASE}/trustgraph-unstructured:${VERSION} .
|
||||
# Combine per-platform images into a multi-arch manifest
|
||||
combine-manifest-%:
|
||||
-@${DOCKER} manifest rm \
|
||||
${CONTAINER_BASE}/trustgraph-${@:combine-manifest-%=%}:${VERSION}
|
||||
${DOCKER} manifest create \
|
||||
${CONTAINER_BASE}/trustgraph-${@:combine-manifest-%=%}:${VERSION} \
|
||||
docker://${CONTAINER_BASE}/trustgraph-${@:combine-manifest-%=%}:${VERSION}-amd64 \
|
||||
docker://${CONTAINER_BASE}/trustgraph-${@:combine-manifest-%=%}:${VERSION}-arm64
|
||||
${DOCKER} manifest push \
|
||||
${CONTAINER_BASE}/trustgraph-${@:combine-manifest-%=%}:${VERSION}
|
||||
|
||||
container-trustgraph-mcp: update-package-versions
|
||||
${DOCKER} build -f containers/Containerfile.mcp -t ${CONTAINER_BASE}/trustgraph-mcp:${VERSION} .
|
||||
# Push a container
|
||||
push-container-%:
|
||||
${DOCKER} push \
|
||||
${CONTAINER_BASE}/trustgraph-${@:push-container-%=%}:${VERSION}
|
||||
|
||||
# Individual container push targets
|
||||
push-trustgraph-base:
|
||||
${DOCKER} push ${CONTAINER_BASE}/trustgraph-base:${VERSION}
|
||||
|
||||
push-trustgraph-flow:
|
||||
${DOCKER} push ${CONTAINER_BASE}/trustgraph-flow:${VERSION}
|
||||
|
||||
push-trustgraph-bedrock:
|
||||
${DOCKER} push ${CONTAINER_BASE}/trustgraph-bedrock:${VERSION}
|
||||
|
||||
push-trustgraph-vertexai:
|
||||
${DOCKER} push ${CONTAINER_BASE}/trustgraph-vertexai:${VERSION}
|
||||
|
||||
push-trustgraph-hf:
|
||||
${DOCKER} push ${CONTAINER_BASE}/trustgraph-hf:${VERSION}
|
||||
|
||||
push-trustgraph-ocr:
|
||||
${DOCKER} push ${CONTAINER_BASE}/trustgraph-ocr:${VERSION}
|
||||
|
||||
push-trustgraph-unstructured:
|
||||
${DOCKER} push ${CONTAINER_BASE}/trustgraph-unstructured:${VERSION}
|
||||
|
||||
push-trustgraph-mcp:
|
||||
${DOCKER} push ${CONTAINER_BASE}/trustgraph-mcp:${VERSION}
|
||||
# Push a manifest (from local multi-arch build)
|
||||
push-manifest-%:
|
||||
${DOCKER} manifest push \
|
||||
${CONTAINER_BASE}/trustgraph-${@:push-manifest-%=%}:${VERSION}
|
||||
|
||||
clean:
|
||||
rm -rf wheels/
|
||||
|
|
@ -164,52 +131,6 @@ clean:
|
|||
set-version:
|
||||
echo '"${VERSION}"' > templates/values/version.jsonnet
|
||||
|
||||
TEMPLATES=azure bedrock claude cohere mix llamafile mistral ollama openai vertexai \
|
||||
openai-neo4j storage
|
||||
|
||||
DCS=$(foreach template,${TEMPLATES},${template:%=tg-launch-%.yaml})
|
||||
|
||||
MODELS=azure bedrock claude cohere llamafile mistral ollama openai vertexai
|
||||
GRAPHS=cassandra neo4j falkordb memgraph
|
||||
|
||||
# tg-launch-%.yaml: templates/%.jsonnet templates/components/version.jsonnet
|
||||
# jsonnet -Jtemplates \
|
||||
# -S ${@:tg-launch-%.yaml=templates/%.jsonnet} > $@
|
||||
|
||||
# VECTORDB=milvus
|
||||
VECTORDB=qdrant
|
||||
|
||||
JSONNET_FLAGS=-J templates -J .
|
||||
|
||||
# Temporarily going back to how templates were built in 0.9 because this
|
||||
# is going away in 0.11.
|
||||
|
||||
update-templates: update-dcs
|
||||
|
||||
JSON_TO_YAML=python -c 'import sys, yaml, json; j=json.loads(sys.stdin.read()); print(yaml.safe_dump(j))'
|
||||
|
||||
update-dcs: set-version
|
||||
for graph in ${GRAPHS}; do \
|
||||
cm=$${graph},pulsar,${VECTORDB},grafana; \
|
||||
input=templates/opts-to-docker-compose.jsonnet; \
|
||||
output=tg-storage-$${graph}.yaml; \
|
||||
echo $${graph} '->' $${output}; \
|
||||
jsonnet ${JSONNET_FLAGS} \
|
||||
--ext-str options=$${cm} $${input} | \
|
||||
${JSON_TO_YAML} > $${output}; \
|
||||
done
|
||||
for model in ${MODELS}; do \
|
||||
for graph in ${GRAPHS}; do \
|
||||
cm=$${graph},pulsar,${VECTORDB},embeddings-hf,graph-rag,grafana,trustgraph,$${model}; \
|
||||
input=templates/opts-to-docker-compose.jsonnet; \
|
||||
output=tg-launch-$${model}-$${graph}.yaml; \
|
||||
echo $${model} + $${graph} '->' $${output}; \
|
||||
jsonnet ${JSONNET_FLAGS} \
|
||||
--ext-str options=$${cm} $${input} | \
|
||||
${JSON_TO_YAML} > $${output}; \
|
||||
done; \
|
||||
done
|
||||
|
||||
docker-hub-login:
|
||||
cat docker-token.txt | \
|
||||
${DOCKER} login -u trustgraph --password-stdin registry-1.docker.io
|
||||
|
|
|
|||
|
|
@ -1,22 +1,22 @@
|
|||
|
||||
# ----------------------------------------------------------------------------
|
||||
# Build an AI container. This does the torch install which is huge, and I
|
||||
# like to avoid re-doing this.
|
||||
# ----------------------------------------------------------------------------
|
||||
# Torch is stable and compiles for ARM64 and AMD64 on Python 3.12
|
||||
|
||||
FROM docker.io/fedora:42 AS ai
|
||||
|
||||
ENV PIP_BREAK_SYSTEM_PACKAGES=1
|
||||
|
||||
RUN dnf install -y python3.13 && \
|
||||
alternatives --install /usr/bin/python python /usr/bin/python3.13 1 && \
|
||||
RUN dnf install -y python3.12 && \
|
||||
alternatives --install /usr/bin/python python /usr/bin/python3.12 1 && \
|
||||
python -m ensurepip --upgrade && \
|
||||
pip3 install --no-cache-dir build wheel aiohttp && \
|
||||
pip3 install --no-cache-dir pulsar-client==3.7.0 && \
|
||||
dnf clean all
|
||||
|
||||
RUN pip3 install torch==2.5.1+cpu \
|
||||
--index-url https://download.pytorch.org/whl/cpu
|
||||
# This won't work on ARM
|
||||
#RUN pip3 install torch==2.5.1+cpu \
|
||||
# --index-url https://download.pytorch.org/whl/cpu
|
||||
|
||||
RUN pip3 install torch
|
||||
|
||||
RUN pip3 install --no-cache-dir \
|
||||
langchain==0.3.25 langchain-core==0.3.60 langchain-huggingface==0.2.0 \
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue