mirror of
https://github.com/trustgraph-ai/trustgraph.git
synced 2026-05-25 23:35:12 +02:00
Add multi-arch (amd64/arm64) container builds and parallel CI (#807)
Pull requests back-ported to release/v2.2: 801, 802, 804, 805
Restructure container builds for multi-platform support, enabling
ARM-based deployments (e.g. Apple Silicon via Docker Desktop).
Makefile:
- Replace per-container named targets with pattern rules
(container-%, manifest-%, platform-%-{amd64,arm64},
combine-manifest-%)
- Add parallel CI targets: platform builds push per-arch images,
combine-manifest creates and pushes the multi-arch manifest list
- Remove legacy cruft targets (update-dcs, update-templates)
CI (release.yaml):
- Split single deploy job into build-platform-image (16 parallel
jobs: 8 containers x 2 platforms) and combine-manifests (8 jobs,
metadata only)
- Use native ARM runners (ubuntu-24.04-arm)
Containerfile.hf:
- Downgrade to Python 3.12 (PyTorch lacks arm64 wheels for 3.13)
- Use standard PyTorch package instead of +cpu variant (no arm64 wheels
on the cpu index)
This commit is contained in:
cybermaggedon
GitHub
parent
dfb6d26a56
commit
b2e69cfa48
3 changed files with 121 additions and 155 deletions
181
Makefile
181
Makefile
|
|
@ -52,51 +52,12 @@ update-package-versions:
|
|||
echo __version__ = \"${VERSION}\" > trustgraph/trustgraph/trustgraph_version.py
|
||||
echo __version__ = \"${VERSION}\" > trustgraph-mcp/trustgraph/mcp_version.py
|
||||
|
||||
FORCE:
|
||||
containers: container-base container-flow \
|
||||
container-bedrock container-vertexai \
|
||||
container-hf container-ocr \
|
||||
container-unstructured container-mcp
|
||||
|
||||
containers: FORCE
|
||||
${DOCKER} build -f containers/Containerfile.base \
|
||||
-t ${CONTAINER_BASE}/trustgraph-base:${VERSION} .
|
||||
${DOCKER} build -f containers/Containerfile.flow \
|
||||
-t ${CONTAINER_BASE}/trustgraph-flow:${VERSION} .
|
||||
${DOCKER} build -f containers/Containerfile.bedrock \
|
||||
-t ${CONTAINER_BASE}/trustgraph-bedrock:${VERSION} .
|
||||
${DOCKER} build -f containers/Containerfile.vertexai \
|
||||
-t ${CONTAINER_BASE}/trustgraph-vertexai:${VERSION} .
|
||||
${DOCKER} build -f containers/Containerfile.hf \
|
||||
-t ${CONTAINER_BASE}/trustgraph-hf:${VERSION} .
|
||||
${DOCKER} build -f containers/Containerfile.ocr \
|
||||
-t ${CONTAINER_BASE}/trustgraph-ocr:${VERSION} .
|
||||
${DOCKER} build -f containers/Containerfile.unstructured \
|
||||
-t ${CONTAINER_BASE}/trustgraph-unstructured:${VERSION} .
|
||||
${DOCKER} build -f containers/Containerfile.mcp \
|
||||
-t ${CONTAINER_BASE}/trustgraph-mcp:${VERSION} .
|
||||
|
||||
some-containers:
|
||||
${DOCKER} build -f containers/Containerfile.base \
|
||||
-t ${CONTAINER_BASE}/trustgraph-base:${VERSION} .
|
||||
${DOCKER} build -f containers/Containerfile.flow \
|
||||
-t ${CONTAINER_BASE}/trustgraph-flow:${VERSION} .
|
||||
# ${DOCKER} build -f containers/Containerfile.unstructured \
|
||||
# -t ${CONTAINER_BASE}/trustgraph-unstructured:${VERSION} .
|
||||
# ${DOCKER} build -f containers/Containerfile.vertexai \
|
||||
# -t ${CONTAINER_BASE}/trustgraph-vertexai:${VERSION} .
|
||||
# ${DOCKER} build -f containers/Containerfile.mcp \
|
||||
# -t ${CONTAINER_BASE}/trustgraph-mcp:${VERSION} .
|
||||
# ${DOCKER} build -f containers/Containerfile.vertexai \
|
||||
# -t ${CONTAINER_BASE}/trustgraph-vertexai:${VERSION} .
|
||||
# ${DOCKER} build -f containers/Containerfile.bedrock \
|
||||
# -t ${CONTAINER_BASE}/trustgraph-bedrock:${VERSION} .
|
||||
|
||||
basic-containers: update-package-versions
|
||||
${DOCKER} build -f containers/Containerfile.base \
|
||||
-t ${CONTAINER_BASE}/trustgraph-base:${VERSION} .
|
||||
${DOCKER} build -f containers/Containerfile.flow \
|
||||
-t ${CONTAINER_BASE}/trustgraph-flow:${VERSION} .
|
||||
|
||||
container.ocr:
|
||||
${DOCKER} build -f containers/Containerfile.ocr \
|
||||
-t ${CONTAINER_BASE}/trustgraph-ocr:${VERSION} .
|
||||
some-containers: container-base container-flow
|
||||
|
||||
push:
|
||||
${DOCKER} push ${CONTAINER_BASE}/trustgraph-base:${VERSION}
|
||||
|
|
@ -109,54 +70,60 @@ push:
|
|||
${DOCKER} push ${CONTAINER_BASE}/trustgraph-mcp:${VERSION}
|
||||
|
||||
# Individual container build targets
|
||||
container-trustgraph-base: update-package-versions
|
||||
${DOCKER} build -f containers/Containerfile.base -t ${CONTAINER_BASE}/trustgraph-base:${VERSION} .
|
||||
container-%: update-package-versions
|
||||
${DOCKER} build \
|
||||
-f containers/Containerfile.${@:container-%=%} \
|
||||
-t ${CONTAINER_BASE}/trustgraph-${@:container-%=%}:${VERSION} .
|
||||
|
||||
container-trustgraph-flow: update-package-versions
|
||||
${DOCKER} build -f containers/Containerfile.flow -t ${CONTAINER_BASE}/trustgraph-flow:${VERSION} .
|
||||
# Multi-arch: build both platforms sequentially into one manifest (local use)
|
||||
manifest-%: update-package-versions
|
||||
-@${DOCKER} manifest rm \
|
||||
${CONTAINER_BASE}/trustgraph-${@:manifest-%=%}:${VERSION}
|
||||
${DOCKER} build --platform linux/amd64,linux/arm64 \
|
||||
-f containers/Containerfile.${@:manifest-%=%} \
|
||||
--manifest \
|
||||
${CONTAINER_BASE}/trustgraph-${@:manifest-%=%}:${VERSION} .
|
||||
|
||||
container-trustgraph-bedrock: update-package-versions
|
||||
${DOCKER} build -f containers/Containerfile.bedrock -t ${CONTAINER_BASE}/trustgraph-bedrock:${VERSION} .
|
||||
# Multi-arch: build a single platform image (for parallel CI)
|
||||
platform-%-amd64: update-package-versions
|
||||
${DOCKER} build --platform linux/amd64 \
|
||||
-f containers/Containerfile.${@:platform-%-amd64=%} \
|
||||
-t ${CONTAINER_BASE}/trustgraph-${@:platform-%-amd64=%}:${VERSION}-amd64 .
|
||||
|
||||
container-trustgraph-vertexai: update-package-versions
|
||||
${DOCKER} build -f containers/Containerfile.vertexai -t ${CONTAINER_BASE}/trustgraph-vertexai:${VERSION} .
|
||||
platform-%-arm64: update-package-versions
|
||||
${DOCKER} build --platform linux/arm64 \
|
||||
-f containers/Containerfile.${@:platform-%-arm64=%} \
|
||||
-t ${CONTAINER_BASE}/trustgraph-${@:platform-%-arm64=%}:${VERSION}-arm64 .
|
||||
|
||||
container-trustgraph-hf: update-package-versions
|
||||
${DOCKER} build -f containers/Containerfile.hf -t ${CONTAINER_BASE}/trustgraph-hf:${VERSION} .
|
||||
# Push a single platform image
|
||||
push-platform-%-amd64:
|
||||
${DOCKER} push \
|
||||
${CONTAINER_BASE}/trustgraph-${@:push-platform-%-amd64=%}:${VERSION}-amd64
|
||||
|
||||
container-trustgraph-ocr: update-package-versions
|
||||
${DOCKER} build -f containers/Containerfile.ocr -t ${CONTAINER_BASE}/trustgraph-ocr:${VERSION} .
|
||||
push-platform-%-arm64:
|
||||
${DOCKER} push \
|
||||
${CONTAINER_BASE}/trustgraph-${@:push-platform-%-arm64=%}:${VERSION}-arm64
|
||||
|
||||
container-trustgraph-unstructured: update-package-versions
|
||||
${DOCKER} build -f containers/Containerfile.unstructured -t ${CONTAINER_BASE}/trustgraph-unstructured:${VERSION} .
|
||||
# Combine per-platform images into a multi-arch manifest
|
||||
combine-manifest-%:
|
||||
-@${DOCKER} manifest rm \
|
||||
${CONTAINER_BASE}/trustgraph-${@:combine-manifest-%=%}:${VERSION}
|
||||
${DOCKER} manifest create \
|
||||
${CONTAINER_BASE}/trustgraph-${@:combine-manifest-%=%}:${VERSION} \
|
||||
docker://${CONTAINER_BASE}/trustgraph-${@:combine-manifest-%=%}:${VERSION}-amd64 \
|
||||
docker://${CONTAINER_BASE}/trustgraph-${@:combine-manifest-%=%}:${VERSION}-arm64
|
||||
${DOCKER} manifest push \
|
||||
${CONTAINER_BASE}/trustgraph-${@:combine-manifest-%=%}:${VERSION}
|
||||
|
||||
container-trustgraph-mcp: update-package-versions
|
||||
${DOCKER} build -f containers/Containerfile.mcp -t ${CONTAINER_BASE}/trustgraph-mcp:${VERSION} .
|
||||
# Push a container
|
||||
push-container-%:
|
||||
${DOCKER} push \
|
||||
${CONTAINER_BASE}/trustgraph-${@:push-container-%=%}:${VERSION}
|
||||
|
||||
# Individual container push targets
|
||||
push-trustgraph-base:
|
||||
${DOCKER} push ${CONTAINER_BASE}/trustgraph-base:${VERSION}
|
||||
|
||||
push-trustgraph-flow:
|
||||
${DOCKER} push ${CONTAINER_BASE}/trustgraph-flow:${VERSION}
|
||||
|
||||
push-trustgraph-bedrock:
|
||||
${DOCKER} push ${CONTAINER_BASE}/trustgraph-bedrock:${VERSION}
|
||||
|
||||
push-trustgraph-vertexai:
|
||||
${DOCKER} push ${CONTAINER_BASE}/trustgraph-vertexai:${VERSION}
|
||||
|
||||
push-trustgraph-hf:
|
||||
${DOCKER} push ${CONTAINER_BASE}/trustgraph-hf:${VERSION}
|
||||
|
||||
push-trustgraph-ocr:
|
||||
${DOCKER} push ${CONTAINER_BASE}/trustgraph-ocr:${VERSION}
|
||||
|
||||
push-trustgraph-unstructured:
|
||||
${DOCKER} push ${CONTAINER_BASE}/trustgraph-unstructured:${VERSION}
|
||||
|
||||
push-trustgraph-mcp:
|
||||
${DOCKER} push ${CONTAINER_BASE}/trustgraph-mcp:${VERSION}
|
||||
# Push a manifest (from local multi-arch build)
|
||||
push-manifest-%:
|
||||
${DOCKER} manifest push \
|
||||
${CONTAINER_BASE}/trustgraph-${@:push-manifest-%=%}:${VERSION}
|
||||
|
||||
clean:
|
||||
rm -rf wheels/
|
||||
|
|
@ -164,52 +131,6 @@ clean:
|
|||
set-version:
|
||||
echo '"${VERSION}"' > templates/values/version.jsonnet
|
||||
|
||||
TEMPLATES=azure bedrock claude cohere mix llamafile mistral ollama openai vertexai \
|
||||
openai-neo4j storage
|
||||
|
||||
DCS=$(foreach template,${TEMPLATES},${template:%=tg-launch-%.yaml})
|
||||
|
||||
MODELS=azure bedrock claude cohere llamafile mistral ollama openai vertexai
|
||||
GRAPHS=cassandra neo4j falkordb memgraph
|
||||
|
||||
# tg-launch-%.yaml: templates/%.jsonnet templates/components/version.jsonnet
|
||||
# jsonnet -Jtemplates \
|
||||
# -S ${@:tg-launch-%.yaml=templates/%.jsonnet} > $@
|
||||
|
||||
# VECTORDB=milvus
|
||||
VECTORDB=qdrant
|
||||
|
||||
JSONNET_FLAGS=-J templates -J .
|
||||
|
||||
# Temporarily going back to how templates were built in 0.9 because this
|
||||
# is going away in 0.11.
|
||||
|
||||
update-templates: update-dcs
|
||||
|
||||
JSON_TO_YAML=python -c 'import sys, yaml, json; j=json.loads(sys.stdin.read()); print(yaml.safe_dump(j))'
|
||||
|
||||
update-dcs: set-version
|
||||
for graph in ${GRAPHS}; do \
|
||||
cm=$${graph},pulsar,${VECTORDB},grafana; \
|
||||
input=templates/opts-to-docker-compose.jsonnet; \
|
||||
output=tg-storage-$${graph}.yaml; \
|
||||
echo $${graph} '->' $${output}; \
|
||||
jsonnet ${JSONNET_FLAGS} \
|
||||
--ext-str options=$${cm} $${input} | \
|
||||
${JSON_TO_YAML} > $${output}; \
|
||||
done
|
||||
for model in ${MODELS}; do \
|
||||
for graph in ${GRAPHS}; do \
|
||||
cm=$${graph},pulsar,${VECTORDB},embeddings-hf,graph-rag,grafana,trustgraph,$${model}; \
|
||||
input=templates/opts-to-docker-compose.jsonnet; \
|
||||
output=tg-launch-$${model}-$${graph}.yaml; \
|
||||
echo $${model} + $${graph} '->' $${output}; \
|
||||
jsonnet ${JSONNET_FLAGS} \
|
||||
--ext-str options=$${cm} $${input} | \
|
||||
${JSON_TO_YAML} > $${output}; \
|
||||
done; \
|
||||
done
|
||||
|
||||
docker-hub-login:
|
||||
cat docker-token.txt | \
|
||||
${DOCKER} login -u trustgraph --password-stdin registry-1.docker.io
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue